diff mbox series

[v1,bpf-next,1/9] bpftool: add testing skeleton

Message ID 20231116194236.1345035-2-chantr4@gmail.com (mailing list archive)
State Changes Requested
Delegated to: BPF
Headers show
Series bpftool: Add end-to-end testing | expand

Checks

Context Check Description
netdev/series_format success Posting correctly formatted
netdev/tree_selection success Clearly marked for bpf-next
netdev/fixes_present success Fixes tag not required for -next series
netdev/header_inline success No static functions without inline keyword in header files
netdev/build_32bit success Errors and warnings before: 8 this patch: 8
netdev/cc_maintainers warning 4 maintainers not CCed: yonghong.song@linux.dev linux-kselftest@vger.kernel.org mykolal@fb.com shuah@kernel.org
netdev/build_clang success Errors and warnings before: 8 this patch: 8
netdev/verify_signedoff success Signed-off-by tag matches author and committer
netdev/deprecated_api success None detected
netdev/check_selftest success No net selftest shell script
netdev/verify_fixes success No Fixes tag
netdev/build_allmodconfig_warn success Errors and warnings before: 8 this patch: 8
netdev/checkpatch warning WARNING: added, moved or deleted file(s), does MAINTAINERS need updating? WARNING: line length of 100 exceeds 80 columns
netdev/build_clang_rust fail Link
netdev/kdoc success Errors and warnings before: 0 this patch: 0
netdev/source_inline success Was 0 now: 0
bpf/vmtest-bpf-next-PR success PR summary
bpf/vmtest-bpf-next-VM_Test-12 success Logs for s390x-gcc / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on s390x with gcc
bpf/vmtest-bpf-next-VM_Test-0 success Logs for Lint
bpf/vmtest-bpf-next-VM_Test-3 success Logs for aarch64-gcc / build / build for aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-2 success Logs for Validate matrix.py
bpf/vmtest-bpf-next-VM_Test-1 success Logs for ShellCheck
bpf/vmtest-bpf-next-VM_Test-10 success Logs for set-matrix
bpf/vmtest-bpf-next-VM_Test-8 success Logs for aarch64-gcc / veristat
bpf/vmtest-bpf-next-VM_Test-11 success Logs for x86_64-gcc / build / build for x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-4 success Logs for aarch64-gcc / test (test_maps, false, 360) / test_maps on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-7 success Logs for aarch64-gcc / test (test_verifier, false, 360) / test_verifier on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-6 success Logs for aarch64-gcc / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-9 success Logs for s390x-gcc / build / build for s390x with gcc
bpf/vmtest-bpf-next-VM_Test-14 success Logs for s390x-gcc / veristat
bpf/vmtest-bpf-next-VM_Test-15 success Logs for set-matrix
bpf/vmtest-bpf-next-VM_Test-16 success Logs for x86_64-gcc / build / build for x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-17 success Logs for x86_64-gcc / test (test_maps, false, 360) / test_maps on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-20 success Logs for x86_64-gcc / test (test_progs_no_alu32_parallel, true, 30) / test_progs_no_alu32_parallel on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-21 success Logs for x86_64-gcc / test (test_progs_parallel, true, 30) / test_progs_parallel on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-22 success Logs for x86_64-gcc / test (test_verifier, false, 360) / test_verifier on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-23 success Logs for x86_64-gcc / veristat / veristat on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-24 success Logs for x86_64-llvm-16 / build / build for x86_64 with llvm-16
bpf/vmtest-bpf-next-VM_Test-28 success Logs for x86_64-llvm-16 / test (test_verifier, false, 360) / test_verifier on x86_64 with llvm-16
bpf/vmtest-bpf-next-VM_Test-29 success Logs for x86_64-llvm-16 / veristat
bpf/vmtest-bpf-next-VM_Test-5 success Logs for aarch64-gcc / test (test_progs, false, 360) / test_progs on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-18 success Logs for x86_64-gcc / test (test_progs, false, 360) / test_progs on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-19 success Logs for x86_64-gcc / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-25 success Logs for x86_64-llvm-16 / test (test_maps, false, 360) / test_maps on x86_64 with llvm-16
bpf/vmtest-bpf-next-VM_Test-26 success Logs for x86_64-llvm-16 / test (test_progs, false, 360) / test_progs on x86_64 with llvm-16
bpf/vmtest-bpf-next-VM_Test-27 success Logs for x86_64-llvm-16 / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on x86_64 with llvm-16
bpf/vmtest-bpf-next-VM_Test-13 success Logs for s390x-gcc / test (test_verifier, false, 360) / test_verifier on s390x with gcc

Commit Message

Manu Bretelle Nov. 16, 2023, 7:42 p.m. UTC
Add minimal cargo project to test bpftool.
An environment variable `BPFTOOL_PATH` can be used to provide the path
to bpftool, defaulting to /usr/sbin/bpftool

    $ cargo check --tests
        Finished dev [unoptimized + debuginfo] target(s) in 0.00s
    $ cargo clippy --tests
        Finished dev [unoptimized + debuginfo] target(s) in 0.05s
    $ BPFTOOL_PATH='../bpftool' cargo test -- --nocapture
        Finished test [unoptimized + debuginfo] target(s) in 0.05s
         Running unittests src/main.rs
    (target/debug/deps/bpftool_tests-172b867215e9364e)

    running 1 test
    Running command "../bpftool" "version"
    test bpftool_tests::run_bpftool ... ok

    test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered
    out; finished in 0.00s

Signed-off-by: Manu Bretelle <chantr4@gmail.com>
---
 .../selftests/bpf/bpftool_tests/.gitignore    |  3 +++
 .../selftests/bpf/bpftool_tests/Cargo.toml    |  4 ++++
 .../bpf/bpftool_tests/src/bpftool_tests.rs    | 20 +++++++++++++++++++
 .../selftests/bpf/bpftool_tests/src/main.rs   |  3 +++
 4 files changed, 30 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/bpftool_tests/.gitignore
 create mode 100644 tools/testing/selftests/bpf/bpftool_tests/Cargo.toml
 create mode 100644 tools/testing/selftests/bpf/bpftool_tests/src/bpftool_tests.rs
 create mode 100644 tools/testing/selftests/bpf/bpftool_tests/src/main.rs

Comments

Alexei Starovoitov Nov. 21, 2023, 1:37 a.m. UTC | #1
On Thu, Nov 16, 2023 at 11:43 AM Manu Bretelle <chantr4@gmail.com> wrote:
>
> +++ b/tools/testing/selftests/bpf/bpftool_tests/src/bpftool_tests.rs
> @@ -0,0 +1,20 @@
> +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
> +use std::process::Command;
> +
> +const BPFTOOL_PATH_ENV: &str = "BPFTOOL_PATH";
> +const BPFTOOL_PATH: &str = "/usr/sbin/bpftool";
> +
> +/// Run a bpftool command and returns the output
> +fn run_bpftool_command(args: &[&str]) -> std::process::Output {
> +    let mut cmd = Command::new(std::env::var(BPFTOOL_PATH_ENV).unwrap_or(BPFTOOL_PATH.to_string()));
> +    cmd.args(args);
> +    println!("Running command {:?}", cmd);
> +    cmd.output().expect("failed to execute process")
> +}
> +
> +/// Simple test to make sure we can run bpftool
> +#[test]
> +fn run_bpftool() {
> +    let output = run_bpftool_command(&["version"]);
> +    assert!(output.status.success());
> +}
> diff --git a/tools/testing/selftests/bpf/bpftool_tests/src/main.rs b/tools/testing/selftests/bpf/bpftool_tests/src/main.rs
> new file mode 100644
> index 000000000000..6b4ffcde7406
> --- /dev/null
> +++ b/tools/testing/selftests/bpf/bpftool_tests/src/main.rs
> @@ -0,0 +1,3 @@
> +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
> +#[cfg(test)]
> +mod bpftool_tests;

There is rust in the kernel tree already.
Most of it is #![no_std] and the rest depend on
rust_is_available.sh and the kernel build system.
This rust usage doesn't fit into two existing rust categories afaics.

Does it have to leave in the kernel tree?
We have bpftool on github, maybe it can be there?
Do you want to run bpftool tester as part of BPF CI and that's why
you want it to be in the kernel tree?
Quentin Monnet Nov. 21, 2023, 4:26 p.m. UTC | #2
2023-11-16 19:43 UTC+0000 ~ Manu Bretelle <chantr4@gmail.com>
> Add minimal cargo project to test bpftool.
> An environment variable `BPFTOOL_PATH` can be used to provide the path
> to bpftool, defaulting to /usr/sbin/bpftool
> 
>     $ cargo check --tests
>         Finished dev [unoptimized + debuginfo] target(s) in 0.00s
>     $ cargo clippy --tests
>         Finished dev [unoptimized + debuginfo] target(s) in 0.05s
>     $ BPFTOOL_PATH='../bpftool' cargo test -- --nocapture
>         Finished test [unoptimized + debuginfo] target(s) in 0.05s
>          Running unittests src/main.rs
>     (target/debug/deps/bpftool_tests-172b867215e9364e)
> 
>     running 1 test
>     Running command "../bpftool" "version"
>     test bpftool_tests::run_bpftool ... ok
> 
>     test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered
>     out; finished in 0.00s
> 
> Signed-off-by: Manu Bretelle <chantr4@gmail.com>
> ---
>  .../selftests/bpf/bpftool_tests/.gitignore    |  3 +++
>  .../selftests/bpf/bpftool_tests/Cargo.toml    |  4 ++++
>  .../bpf/bpftool_tests/src/bpftool_tests.rs    | 20 +++++++++++++++++++
>  .../selftests/bpf/bpftool_tests/src/main.rs   |  3 +++
>  4 files changed, 30 insertions(+)
>  create mode 100644 tools/testing/selftests/bpf/bpftool_tests/.gitignore
>  create mode 100644 tools/testing/selftests/bpf/bpftool_tests/Cargo.toml
>  create mode 100644 tools/testing/selftests/bpf/bpftool_tests/src/bpftool_tests.rs
>  create mode 100644 tools/testing/selftests/bpf/bpftool_tests/src/main.rs
> 
> diff --git a/tools/testing/selftests/bpf/bpftool_tests/.gitignore b/tools/testing/selftests/bpf/bpftool_tests/.gitignore
> new file mode 100644
> index 000000000000..cf8177c6aabd
> --- /dev/null
> +++ b/tools/testing/selftests/bpf/bpftool_tests/.gitignore
> @@ -0,0 +1,3 @@
> +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
> +/target/**
> +/Cargo.lock
> diff --git a/tools/testing/selftests/bpf/bpftool_tests/Cargo.toml b/tools/testing/selftests/bpf/bpftool_tests/Cargo.toml
> new file mode 100644
> index 000000000000..34df3002003f
> --- /dev/null
> +++ b/tools/testing/selftests/bpf/bpftool_tests/Cargo.toml
> @@ -0,0 +1,4 @@
> +[package]
> +name = "bpftool_tests"
> +version = "0.1.0"
> +edition = "2021"
> diff --git a/tools/testing/selftests/bpf/bpftool_tests/src/bpftool_tests.rs b/tools/testing/selftests/bpf/bpftool_tests/src/bpftool_tests.rs
> new file mode 100644
> index 000000000000..251dbf3861fe
> --- /dev/null
> +++ b/tools/testing/selftests/bpf/bpftool_tests/src/bpftool_tests.rs
> @@ -0,0 +1,20 @@
> +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause

Any particular reason for this particular choice of license? If not,
could you please reconsider? All sources for bpftool use (GPL-2.0-only
OR BSD-2-Clause), as you use in the .gitignore above, so it would make
sense to have the related tests with the same licenses. It would
certainly make things easier if someone need to ship the tests along
with the sources in the future.

(Same comment for the other Rust files you add in this commit and the next.)

> +use std::process::Command;
> +
> +const BPFTOOL_PATH_ENV: &str = "BPFTOOL_PATH";
> +const BPFTOOL_PATH: &str = "/usr/sbin/bpftool";

This is a decent choice given that it's where the binary will likely end
up for most distributions, but I'd maybe use "/usr/local/sbin/bpftool"
instead to remain consistent with the prefix in bpftool's Makefile, and
default to where we install bpftool when we "make install" it.
Quentin Monnet Nov. 21, 2023, 4:26 p.m. UTC | #3
2023-11-21 01:38 UTC+0000 ~ Alexei Starovoitov
<alexei.starovoitov@gmail.com>
> On Thu, Nov 16, 2023 at 11:43 AM Manu Bretelle <chantr4@gmail.com> wrote:
>>
>> +++ b/tools/testing/selftests/bpf/bpftool_tests/src/bpftool_tests.rs
>> @@ -0,0 +1,20 @@
>> +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
>> +use std::process::Command;
>> +
>> +const BPFTOOL_PATH_ENV: &str = "BPFTOOL_PATH";
>> +const BPFTOOL_PATH: &str = "/usr/sbin/bpftool";
>> +
>> +/// Run a bpftool command and returns the output
>> +fn run_bpftool_command(args: &[&str]) -> std::process::Output {
>> +    let mut cmd = Command::new(std::env::var(BPFTOOL_PATH_ENV).unwrap_or(BPFTOOL_PATH.to_string()));
>> +    cmd.args(args);
>> +    println!("Running command {:?}", cmd);
>> +    cmd.output().expect("failed to execute process")
>> +}
>> +
>> +/// Simple test to make sure we can run bpftool
>> +#[test]
>> +fn run_bpftool() {
>> +    let output = run_bpftool_command(&["version"]);
>> +    assert!(output.status.success());
>> +}
>> diff --git a/tools/testing/selftests/bpf/bpftool_tests/src/main.rs b/tools/testing/selftests/bpf/bpftool_tests/src/main.rs
>> new file mode 100644
>> index 000000000000..6b4ffcde7406
>> --- /dev/null
>> +++ b/tools/testing/selftests/bpf/bpftool_tests/src/main.rs
>> @@ -0,0 +1,3 @@
>> +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
>> +#[cfg(test)]
>> +mod bpftool_tests;
> 
> There is rust in the kernel tree already.
> Most of it is #![no_std] and the rest depend on
> rust_is_available.sh and the kernel build system.
> This rust usage doesn't fit into two existing rust categories afaics.

I haven't followed closely the introduction of Rust in the kernel
repository, so apologies if I'm incorrect. From what I understand,
#![no_std] is to avoid linking against the std-crate, which is necessary
for Rust code that needs to be compiled as part of the kernel or
modules, but is maybe not relevant for something external like a test
suite? As for rust_is_available.sh, we would need a way to determine
whether Rust is available indeed, before plugging these tests into the
Makefile for the BPF selftests.

As far as I'm aware, these would be the first selftests written in Rust
in the repo (other than for the code under rust/). I'm fine having tests
in Rust for bpftool, for what it matters. Whether we want selftests in
Rust in the kernel repo is another thing.

> 
> Does it have to leave in the kernel tree?
> We have bpftool on github, maybe it can be there?
> Do you want to run bpftool tester as part of BPF CI and that's why
> you want it to be in the kernel tree?

It doesn't _have_ to be in the kernel tree, although it's a nice place
where to have it. We have bpftool on GitHub, but the CI that runs there
is triggered only when syncing the mirror to check that mirroring is not
broken, so after new patches are applied to bpf-next. If we want this on
GitHub, we would rather target the BPF CI infra.

A nice point of having it in the repo would be the ability to add tests
at the same time as we add features in bpftool, of course.

Quentin
Alexei Starovoitov Nov. 21, 2023, 4:42 p.m. UTC | #4
On Tue, Nov 21, 2023 at 8:26 AM Quentin Monnet <quentin@isovalent.com> wrote:
>
> >
> > Does it have to leave in the kernel tree?
> > We have bpftool on github, maybe it can be there?
> > Do you want to run bpftool tester as part of BPF CI and that's why
> > you want it to be in the kernel tree?
>
> It doesn't _have_ to be in the kernel tree, although it's a nice place
> where to have it. We have bpftool on GitHub, but the CI that runs there
> is triggered only when syncing the mirror to check that mirroring is not
> broken, so after new patches are applied to bpf-next. If we want this on
> GitHub, we would rather target the BPF CI infra.
>
> A nice point of having it in the repo would be the ability to add tests
> at the same time as we add features in bpftool, of course.

Sounds nice in theory, but in practice that would mean that
every bpftool developer adding a new feature would need to learn rust
to add a corresponding test?
I suspect devs might object to such a requirement.
If tester and bpftool are not sync then they can be in separate repos.
Andrii Nakryiko Nov. 21, 2023, 7:50 p.m. UTC | #5
On Tue, Nov 21, 2023 at 8:42 AM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Tue, Nov 21, 2023 at 8:26 AM Quentin Monnet <quentin@isovalent.com> wrote:
> >
> > >
> > > Does it have to leave in the kernel tree?
> > > We have bpftool on github, maybe it can be there?
> > > Do you want to run bpftool tester as part of BPF CI and that's why
> > > you want it to be in the kernel tree?
> >
> > It doesn't _have_ to be in the kernel tree, although it's a nice place
> > where to have it. We have bpftool on GitHub, but the CI that runs there
> > is triggered only when syncing the mirror to check that mirroring is not
> > broken, so after new patches are applied to bpf-next. If we want this on
> > GitHub, we would rather target the BPF CI infra.
> >
> > A nice point of having it in the repo would be the ability to add tests
> > at the same time as we add features in bpftool, of course.
>
> Sounds nice in theory, but in practice that would mean that
> every bpftool developer adding a new feature would need to learn rust
> to add a corresponding test?
> I suspect devs might object to such a requirement.
> If tester and bpftool are not sync then they can be in separate repos.

I'm also wondering what Rust and libbpf-rs dependency adds here? It
feels like bash+jq or Python script should be able to "drive" bpftool
CLI and validate output, no?
Quentin Monnet Nov. 27, 2023, 5:07 p.m. UTC | #6
2023-11-21 19:50 UTC+0000 ~ Andrii Nakryiko <andrii.nakryiko@gmail.com>
> On Tue, Nov 21, 2023 at 8:42 AM Alexei Starovoitov
> <alexei.starovoitov@gmail.com> wrote:
>>
>> On Tue, Nov 21, 2023 at 8:26 AM Quentin Monnet <quentin@isovalent.com> wrote:
>>>
>>>>
>>>> Does it have to leave in the kernel tree?
>>>> We have bpftool on github, maybe it can be there?
>>>> Do you want to run bpftool tester as part of BPF CI and that's why
>>>> you want it to be in the kernel tree?
>>>
>>> It doesn't _have_ to be in the kernel tree, although it's a nice place
>>> where to have it. We have bpftool on GitHub, but the CI that runs there
>>> is triggered only when syncing the mirror to check that mirroring is not
>>> broken, so after new patches are applied to bpf-next. If we want this on
>>> GitHub, we would rather target the BPF CI infra.
>>>
>>> A nice point of having it in the repo would be the ability to add tests
>>> at the same time as we add features in bpftool, of course.
>>
>> Sounds nice in theory, but in practice that would mean that
>> every bpftool developer adding a new feature would need to learn rust
>> to add a corresponding test?
>> I suspect devs might object to such a requirement.
>> If tester and bpftool are not sync then they can be in separate repos.
> 
> I'm also wondering what Rust and libbpf-rs dependency adds here? It
> feels like bash+jq or Python script should be able to "drive" bpftool
> CLI and validate output, no?

As I understand, one advantage is to get an easy way to tap into
libbpf's functions to load the objects we need in order to test the
different bpftool features. There are a number of program/map types that
we just cannot load with bpftool at this time, so we need to set up the
objects we need with another loader. Libbpf-rs allows to do that, and
the "cargo test" infra seems like a convenient way to focus on the tests
only. Bash+jq wouldn't allow to load objects unsupported by bpftool, for
example.

Manu, did you have other reasons in mind?
Quentin Monnet Nov. 27, 2023, 5:07 p.m. UTC | #7
2023-11-21 16:42 UTC+0000 ~ Alexei Starovoitov
<alexei.starovoitov@gmail.com>
> On Tue, Nov 21, 2023 at 8:26 AM Quentin Monnet <quentin@isovalent.com> wrote:
>>
>>>
>>> Does it have to leave in the kernel tree?
>>> We have bpftool on github, maybe it can be there?
>>> Do you want to run bpftool tester as part of BPF CI and that's why
>>> you want it to be in the kernel tree?
>>
>> It doesn't _have_ to be in the kernel tree, although it's a nice place
>> where to have it. We have bpftool on GitHub, but the CI that runs there
>> is triggered only when syncing the mirror to check that mirroring is not
>> broken, so after new patches are applied to bpf-next. If we want this on
>> GitHub, we would rather target the BPF CI infra.
>>
>> A nice point of having it in the repo would be the ability to add tests
>> at the same time as we add features in bpftool, of course.
> 
> Sounds nice in theory, but in practice that would mean that
> every bpftool developer adding a new feature would need to learn rust
> to add a corresponding test?
> I suspect devs might object to such a requirement.

True. I've been hoping the tests would look easy enough that devs could
update them without being particularly versed in Rust, but this is
probably wishful thinking, and prone to getting bugs in the tests.

I don't have a good proposal to address this, so I agree, this is
probably not a reasonable requirement.

> If tester and bpftool are not sync then they can be in separate repos.

Makes sense. I'd like to have the tests in the same repo, but for this
time, let's focus on getting these Rust tests added to the BPF CI infra
instead, if there's no easy way to switch to a more consensual language.
Manu, thoughts?

Quentin
Andrii Nakryiko Nov. 27, 2023, 6:39 p.m. UTC | #8
On Mon, Nov 27, 2023 at 9:07 AM Quentin Monnet <quentin@isovalent.com> wrote:
>
> 2023-11-21 19:50 UTC+0000 ~ Andrii Nakryiko <andrii.nakryiko@gmail.com>
> > On Tue, Nov 21, 2023 at 8:42 AM Alexei Starovoitov
> > <alexei.starovoitov@gmail.com> wrote:
> >>
> >> On Tue, Nov 21, 2023 at 8:26 AM Quentin Monnet <quentin@isovalent.com> wrote:
> >>>
> >>>>
> >>>> Does it have to leave in the kernel tree?
> >>>> We have bpftool on github, maybe it can be there?
> >>>> Do you want to run bpftool tester as part of BPF CI and that's why
> >>>> you want it to be in the kernel tree?
> >>>
> >>> It doesn't _have_ to be in the kernel tree, although it's a nice place
> >>> where to have it. We have bpftool on GitHub, but the CI that runs there
> >>> is triggered only when syncing the mirror to check that mirroring is not
> >>> broken, so after new patches are applied to bpf-next. If we want this on
> >>> GitHub, we would rather target the BPF CI infra.
> >>>
> >>> A nice point of having it in the repo would be the ability to add tests
> >>> at the same time as we add features in bpftool, of course.
> >>
> >> Sounds nice in theory, but in practice that would mean that
> >> every bpftool developer adding a new feature would need to learn rust
> >> to add a corresponding test?
> >> I suspect devs might object to such a requirement.
> >> If tester and bpftool are not sync then they can be in separate repos.
> >
> > I'm also wondering what Rust and libbpf-rs dependency adds here? It
> > feels like bash+jq or Python script should be able to "drive" bpftool
> > CLI and validate output, no?
>
> As I understand, one advantage is to get an easy way to tap into
> libbpf's functions to load the objects we need in order to test the
> different bpftool features. There are a number of program/map types that
> we just cannot load with bpftool at this time, so we need to set up the
> objects we need with another loader. Libbpf-rs allows to do that, and
> the "cargo test" infra seems like a convenient way to focus on the tests
> only. Bash+jq wouldn't allow to load objects unsupported by bpftool, for
> example.

Can we use veristat to load BPF object files? we might need some
option to auto-pin programs in some directory or something to keep
them live long enough, I suppose, but it's totally in our control.

>
> Manu, did you have other reasons in mind?
Manu Bretelle Dec. 15, 2023, 6:26 a.m. UTC | #9
I am going to start by apologizing for dropping the ball for so long... I
originally planned to get back to this after thanksgiving holidays... but weeks
snowballed one after the other.

On Mon, Nov 27, 2023 at 10:39:34AM -0800, Andrii Nakryiko wrote:
> On Mon, Nov 27, 2023 at 9:07 AM Quentin Monnet <quentin@isovalent.com> wrote:
> >
> > 2023-11-21 19:50 UTC+0000 ~ Andrii Nakryiko <andrii.nakryiko@gmail.com>
> > > On Tue, Nov 21, 2023 at 8:42 AM Alexei Starovoitov
> > > <alexei.starovoitov@gmail.com> wrote:
> > >>
> > >> On Tue, Nov 21, 2023 at 8:26 AM Quentin Monnet <quentin@isovalent.com> wrote:
> > >>>
> > >>>>
> > >>>> Does it have to leave in the kernel tree?
> > >>>> We have bpftool on github, maybe it can be there?
> > >>>> Do you want to run bpftool tester as part of BPF CI and that's why
> > >>>> you want it to be in the kernel tree?
> > >>>
> > >>> It doesn't _have_ to be in the kernel tree, although it's a nice place
> > >>> where to have it. We have bpftool on GitHub, but the CI that runs there
> > >>> is triggered only when syncing the mirror to check that mirroring is not
> > >>> broken, so after new patches are applied to bpf-next. If we want this on
> > >>> GitHub, we would rather target the BPF CI infra.
> > >>>
> > >>> A nice point of having it in the repo would be the ability to add tests
> > >>> at the same time as we add features in bpftool, of course.

Indeed, it does not have to live in the tree, while it could be more convenient
as Quentin highlighted, as much as running it on BPF CI we could be just fine
by having it hosted in a separate repo.
People can always have a clone of the repo and use it to validate the behaviour
has not changed, or changed in expected ways, and have a separate PR if tests
are added. Definitely not as convenient, but likely better than nothing.


> > >>
> > >> Sounds nice in theory, but in practice that would mean that
> > >> every bpftool developer adding a new feature would need to learn rust
> > >> to add a corresponding test?
> > >> I suspect devs might object to such a requirement.
> > >> If tester and bpftool are not sync then they can be in separate repos.
> > >
> > > I'm also wondering what Rust and libbpf-rs dependency adds here? It
> > > feels like bash+jq or Python script should be able to "drive" bpftool
> > > CLI and validate output, no?
> >
> > As I understand, one advantage is to get an easy way to tap into
> > libbpf's functions to load the objects we need in order to test the
> > different bpftool features. There are a number of program/map types that
> > we just cannot load with bpftool at this time, so we need to set up the
> > objects we need with another loader. Libbpf-rs allows to do that, and
> > the "cargo test" infra seems like a convenient way to focus on the tests
> > only. Bash+jq wouldn't allow to load objects unsupported by bpftool, for
> > example.

There were a couple of reasons that you correctly enumerated:
- having a built-in test runner (that could have been other languages)
- libbpf-{cargo,rs} was taking care of the machinery with skeleton, lifecycle of
  a BPF program.
- "native access" to access/manipulate BPF objects from the testing language and
  use bpftool as a blackbox.
- caring about writing the test, not a framework to run them.
- convenience of the rust toolchain to manage depedencies.
- the bells and whistles that come with the language that make formatting/linting
  a no-brainer.
- bash+jq would have probably either limited, or getting overly complex/brittle
  beyond basic checks, and hard to maintain as more tests get added.
- python would have been filling this gap, but without native interaction.

aside from that, another motivation that helped with the choice is that I
originally wrote this as a way to validate bpftool was meeting our requirements
internally as we sync and deploy it, and rust is one of the languages that is
supported to run in our internal vm testing framework.

> 
> Can we use veristat to load BPF object files? we might need some
> option to auto-pin programs in some directory or something to keep
> them live long enough, I suppose, but it's totally in our control.
> 

This probably solves the loading part, but we should also be able to do this
with bpftool too.

> >
> > Manu, did you have other reasons in mind?
Manu Bretelle Dec. 15, 2023, 6:37 a.m. UTC | #10
On Mon, Nov 27, 2023 at 05:07:15PM +0000, Quentin Monnet wrote:
> 2023-11-21 16:42 UTC+0000 ~ Alexei Starovoitov
> <alexei.starovoitov@gmail.com>
> > On Tue, Nov 21, 2023 at 8:26 AM Quentin Monnet <quentin@isovalent.com> wrote:
> >>
> >>>
> >>> Does it have to leave in the kernel tree?
> >>> We have bpftool on github, maybe it can be there?
> >>> Do you want to run bpftool tester as part of BPF CI and that's why
> >>> you want it to be in the kernel tree?
> >>
> >> It doesn't _have_ to be in the kernel tree, although it's a nice place
> >> where to have it. We have bpftool on GitHub, but the CI that runs there
> >> is triggered only when syncing the mirror to check that mirroring is not
> >> broken, so after new patches are applied to bpf-next. If we want this on
> >> GitHub, we would rather target the BPF CI infra.
> >>
> >> A nice point of having it in the repo would be the ability to add tests
> >> at the same time as we add features in bpftool, of course.
> > 
> > Sounds nice in theory, but in practice that would mean that
> > every bpftool developer adding a new feature would need to learn rust
> > to add a corresponding test?
> > I suspect devs might object to such a requirement.
> 
> True. I've been hoping the tests would look easy enough that devs could
> update them without being particularly versed in Rust, but this is
> probably wishful thinking, and prone to getting bugs in the tests.
> 
> I don't have a good proposal to address this, so I agree, this is
> probably not a reasonable requirement.
> 
> > If tester and bpftool are not sync then they can be in separate repos.
> 
> Makes sense. I'd like to have the tests in the same repo, but for this
> time, let's focus on getting these Rust tests added to the BPF CI infra
> instead, if there's no easy way to switch to a more consensual language.
> Manu, thoughts?

I am fine with that, the work I have done cleaning my original code for this
series is (or at least with minimal changes) self-contained.
Having them hosted outside the tree and used is likely better than nothing.
People can still build upon, and experience will help informing if we should
eventually try to merge this back in.


Manu
> 
> Quentin
diff mbox series

Patch

diff --git a/tools/testing/selftests/bpf/bpftool_tests/.gitignore b/tools/testing/selftests/bpf/bpftool_tests/.gitignore
new file mode 100644
index 000000000000..cf8177c6aabd
--- /dev/null
+++ b/tools/testing/selftests/bpf/bpftool_tests/.gitignore
@@ -0,0 +1,3 @@ 
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/target/**
+/Cargo.lock
diff --git a/tools/testing/selftests/bpf/bpftool_tests/Cargo.toml b/tools/testing/selftests/bpf/bpftool_tests/Cargo.toml
new file mode 100644
index 000000000000..34df3002003f
--- /dev/null
+++ b/tools/testing/selftests/bpf/bpftool_tests/Cargo.toml
@@ -0,0 +1,4 @@ 
+[package]
+name = "bpftool_tests"
+version = "0.1.0"
+edition = "2021"
diff --git a/tools/testing/selftests/bpf/bpftool_tests/src/bpftool_tests.rs b/tools/testing/selftests/bpf/bpftool_tests/src/bpftool_tests.rs
new file mode 100644
index 000000000000..251dbf3861fe
--- /dev/null
+++ b/tools/testing/selftests/bpf/bpftool_tests/src/bpftool_tests.rs
@@ -0,0 +1,20 @@ 
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+use std::process::Command;
+
+const BPFTOOL_PATH_ENV: &str = "BPFTOOL_PATH";
+const BPFTOOL_PATH: &str = "/usr/sbin/bpftool";
+
+/// Run a bpftool command and returns the output
+fn run_bpftool_command(args: &[&str]) -> std::process::Output {
+    let mut cmd = Command::new(std::env::var(BPFTOOL_PATH_ENV).unwrap_or(BPFTOOL_PATH.to_string()));
+    cmd.args(args);
+    println!("Running command {:?}", cmd);
+    cmd.output().expect("failed to execute process")
+}
+
+/// Simple test to make sure we can run bpftool
+#[test]
+fn run_bpftool() {
+    let output = run_bpftool_command(&["version"]);
+    assert!(output.status.success());
+}
diff --git a/tools/testing/selftests/bpf/bpftool_tests/src/main.rs b/tools/testing/selftests/bpf/bpftool_tests/src/main.rs
new file mode 100644
index 000000000000..6b4ffcde7406
--- /dev/null
+++ b/tools/testing/selftests/bpf/bpftool_tests/src/main.rs
@@ -0,0 +1,3 @@ 
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+#[cfg(test)]
+mod bpftool_tests;