| Message ID | 291c5437db94057a3b045a6f036b02658380b05b.1685936428.git.falcon@tinylab.org (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | selftests/nolibc: fix up and improve test report | expand |
On Mon, Jun 05, 2023 at 11:48:52AM +0800, Zhangjin Wu wrote: > A standalone test-report target is added to let the run, run-user and > rerun targets share them. > > Signed-off-by: Zhangjin Wu <falcon@tinylab.org> > --- > tools/testing/selftests/nolibc/Makefile | 26 ++++++++++++------------- > 1 file changed, 13 insertions(+), 13 deletions(-) > > diff --git a/tools/testing/selftests/nolibc/Makefile b/tools/testing/selftests/nolibc/Makefile > index be4159837494..8149ace2938a 100644 > --- a/tools/testing/selftests/nolibc/Makefile > +++ b/tools/testing/selftests/nolibc/Makefile > @@ -127,14 +127,18 @@ nolibc-test: nolibc-test.c sysroot/$(ARCH)/include > libc-test: nolibc-test.c > $(QUIET_CC)$(CC) -o $@ $< > > -# qemu user-land test > -run-user: nolibc-test > - $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : > +test-report: > $(Q)awk '/\[OK\]$$/{p++} /\[FAIL\]$$/{f++} /\[SKIPPED\]$$/{s++} \ > END{ printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); \ > if (s+f > 0) printf(" See all results in %s\n", ARGV[1]); else print; }' \ > $(CURDIR)/run.out > > +# qemu user-land test > +_run-user: nolibc-test > + $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : > + > +run-user: _run-user test-report > + This will not reliably work, there's no ordering here, nothing guarantees that test-report will run *after* _run-user (e.g. make -j). Another approach is needed if you want to factor this, but in general creating sequences in makefiles is difficult and often more painful than having 3 times the same 3 lines. Willy
> On Mon, Jun 05, 2023 at 11:48:52AM +0800, Zhangjin Wu wrote: > > A standalone test-report target is added to let the run, run-user and > > rerun targets share them. > > > > Signed-off-by: Zhangjin Wu <falcon@tinylab.org> > > --- > > tools/testing/selftests/nolibc/Makefile | 26 ++++++++++++------------- > > 1 file changed, 13 insertions(+), 13 deletions(-) > > > > diff --git a/tools/testing/selftests/nolibc/Makefile b/tools/testing/selftests/nolibc/Makefile > > index be4159837494..8149ace2938a 100644 > > --- a/tools/testing/selftests/nolibc/Makefile > > +++ b/tools/testing/selftests/nolibc/Makefile > > @@ -127,14 +127,18 @@ nolibc-test: nolibc-test.c sysroot/$(ARCH)/include > > libc-test: nolibc-test.c > > $(QUIET_CC)$(CC) -o $@ $< > > > > -# qemu user-land test > > -run-user: nolibc-test > > - $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : > > +test-report: > > $(Q)awk '/\[OK\]$$/{p++} /\[FAIL\]$$/{f++} /\[SKIPPED\]$$/{s++} \ > > END{ printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); \ > > if (s+f > 0) printf(" See all results in %s\n", ARGV[1]); else print; }' \ > > $(CURDIR)/run.out > > > > +# qemu user-land test > > +_run-user: nolibc-test > > + $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : > > + > > +run-user: _run-user test-report > > + > > This will not reliably work, there's no ordering here, nothing guarantees > that test-report will run *after* _run-user (e.g. make -j). Another > approach is needed if you want to factor this, but in general creating > sequences in makefiles is difficult and often more painful than having > 3 times the same 3 lines. > Ok, thanks, what about this? # LOG_REPORT: report the test results LOG_REPORT := awk '/\[OK\][\r]*$$/{p++} /\[FAIL\][\r]*$$/{f++} /\[SKIPPED\][\r]*$$/{s++} \ END{ printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); \ printf(" See all results in %s\n", ARGV[1]); }' run-user: nolibc-test $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : $(Q)$(LOG_REPORT) $(CURDIR)/run.out run: kernel $(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out" $(Q)$(LOG_REPORT) $(CURDIR)/run.out rerun: $(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out" $(Q)$(LOG_REPORT) $(CURDIR)/run.out Or we directly add a standalone test report script? something like tools/testing/selftests/nolibc/report.sh #!/bin/sh # # report.sh -- report the test results of nolibc-test # LOG_FILE=$1 [ ! -f "$LOG_FILE" ] && echo "Usage: $0 /path/to/run.out" awk ' /\[OK\][\r]*$$/{ p++ } /\[FAIL\][\r]*$$/{ f++ } /\[SKIPPED\][\r]*$$/{ s++ } END { printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); printf(" See all results in %s\n", ARGV[1]); }' $LOG_FILE And use it like this: LOG_REPORT = $(CURDIR)/report.sh Best regards, Zhangjin > Willy
Hi, Willy > > On Mon, Jun 05, 2023 at 11:48:52AM +0800, Zhangjin Wu wrote: > > > A standalone test-report target is added to let the run, run-user and > > > rerun targets share them. > > > > > > Signed-off-by: Zhangjin Wu <falcon@tinylab.org> > > > --- > > > tools/testing/selftests/nolibc/Makefile | 26 ++++++++++++------------- > > > 1 file changed, 13 insertions(+), 13 deletions(-) > > > > > > diff --git a/tools/testing/selftests/nolibc/Makefile b/tools/testing/selftests/nolibc/Makefile > > > index be4159837494..8149ace2938a 100644 > > > --- a/tools/testing/selftests/nolibc/Makefile > > > +++ b/tools/testing/selftests/nolibc/Makefile > > > @@ -127,14 +127,18 @@ nolibc-test: nolibc-test.c sysroot/$(ARCH)/include > > > libc-test: nolibc-test.c > > > $(QUIET_CC)$(CC) -o $@ $< > > > > > > -# qemu user-land test > > > -run-user: nolibc-test > > > - $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : > > > +test-report: > > > $(Q)awk '/\[OK\]$$/{p++} /\[FAIL\]$$/{f++} /\[SKIPPED\]$$/{s++} \ > > > END{ printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); \ > > > if (s+f > 0) printf(" See all results in %s\n", ARGV[1]); else print; }' \ > > > $(CURDIR)/run.out > > > > > > +# qemu user-land test > > > +_run-user: nolibc-test > > > + $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : > > > + > > > +run-user: _run-user test-report > > > + > > > > This will not reliably work, there's no ordering here, nothing guarantees > > that test-report will run *after* _run-user (e.g. make -j). Another > > approach is needed if you want to factor this, but in general creating > > sequences in makefiles is difficult and often more painful than having > > 3 times the same 3 lines. > > > > Ok, thanks, what about this? > > # LOG_REPORT: report the test results > LOG_REPORT := awk '/\[OK\][\r]*$$/{p++} /\[FAIL\][\r]*$$/{f++} /\[SKIPPED\][\r]*$$/{s++} \ > END{ printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); \ > printf(" See all results in %s\n", ARGV[1]); }' > > run-user: nolibc-test > $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : > $(Q)$(LOG_REPORT) $(CURDIR)/run.out > > run: kernel > $(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out" > $(Q)$(LOG_REPORT) $(CURDIR)/run.out > > rerun: > $(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out" > $(Q)$(LOG_REPORT) $(CURDIR)/run.out > > Or we directly add a standalone test report script? something like > tools/testing/selftests/nolibc/report.sh > > #!/bin/sh > # > # report.sh -- report the test results of nolibc-test > # > > LOG_FILE=$1 > [ ! -f "$LOG_FILE" ] && echo "Usage: $0 /path/to/run.out" > > awk ' > /\[OK\][\r]*$$/{ p++ } > /\[FAIL\][\r]*$$/{ f++ } > /\[SKIPPED\][\r]*$$/{ s++ } > > END { > printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); > printf(" See all results in %s\n", ARGV[1]); > }' $LOG_FILE > > And use it like this: > > LOG_REPORT = $(CURDIR)/report.sh > I plan to renew this patchset, which one of the above methods do you prefer? For the always print statement: printf(" See all results in %s\n", ARGV[1]); }' I will paste the reason why I need it, as mentioned in [1], if you still need a clean test report, I will give up this change ;-) Thanks, Zhangjin --- [1]: https://lore.kernel.org/linux-riscv/20230605070508.153407-1-falcon@tinylab.org/ > Best regards, > Zhangjin > > > Willy
Hi Zhangjin, On Wed, Jun 07, 2023 at 01:52:00PM +0800, Zhangjin Wu wrote: > Hi, Willy > > > > On Mon, Jun 05, 2023 at 11:48:52AM +0800, Zhangjin Wu wrote: > > > > A standalone test-report target is added to let the run, run-user and > > > > rerun targets share them. > > > > > > > > Signed-off-by: Zhangjin Wu <falcon@tinylab.org> > > > > --- > > > > tools/testing/selftests/nolibc/Makefile | 26 ++++++++++++------------- > > > > 1 file changed, 13 insertions(+), 13 deletions(-) > > > > > > > > diff --git a/tools/testing/selftests/nolibc/Makefile b/tools/testing/selftests/nolibc/Makefile > > > > index be4159837494..8149ace2938a 100644 > > > > --- a/tools/testing/selftests/nolibc/Makefile > > > > +++ b/tools/testing/selftests/nolibc/Makefile > > > > @@ -127,14 +127,18 @@ nolibc-test: nolibc-test.c sysroot/$(ARCH)/include > > > > libc-test: nolibc-test.c > > > > $(QUIET_CC)$(CC) -o $@ $< > > > > > > > > -# qemu user-land test > > > > -run-user: nolibc-test > > > > - $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : > > > > +test-report: > > > > $(Q)awk '/\[OK\]$$/{p++} /\[FAIL\]$$/{f++} /\[SKIPPED\]$$/{s++} \ > > > > END{ printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); \ > > > > if (s+f > 0) printf(" See all results in %s\n", ARGV[1]); else print; }' \ > > > > $(CURDIR)/run.out > > > > > > > > +# qemu user-land test > > > > +_run-user: nolibc-test > > > > + $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : > > > > + > > > > +run-user: _run-user test-report > > > > + > > > > > > This will not reliably work, there's no ordering here, nothing guarantees > > > that test-report will run *after* _run-user (e.g. make -j). Another > > > approach is needed if you want to factor this, but in general creating > > > sequences in makefiles is difficult and often more painful than having > > > 3 times the same 3 lines. > > > > > > > Ok, thanks, what about this? > > > > # LOG_REPORT: report the test results > > LOG_REPORT := awk '/\[OK\][\r]*$$/{p++} /\[FAIL\][\r]*$$/{f++} /\[SKIPPED\][\r]*$$/{s++} \ > > END{ printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); \ > > printf(" See all results in %s\n", ARGV[1]); }' > > > > run-user: nolibc-test > > $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : > > $(Q)$(LOG_REPORT) $(CURDIR)/run.out > > > > run: kernel > > $(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out" > > $(Q)$(LOG_REPORT) $(CURDIR)/run.out > > > > rerun: > > $(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out" > > $(Q)$(LOG_REPORT) $(CURDIR)/run.out > > > > Or we directly add a standalone test report script? something like > > tools/testing/selftests/nolibc/report.sh > > > > #!/bin/sh > > # > > # report.sh -- report the test results of nolibc-test > > # > > > > LOG_FILE=$1 > > [ ! -f "$LOG_FILE" ] && echo "Usage: $0 /path/to/run.out" > > > > awk ' > > /\[OK\][\r]*$$/{ p++ } > > /\[FAIL\][\r]*$$/{ f++ } > > /\[SKIPPED\][\r]*$$/{ s++ } > > > > END { > > printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); > > printf(" See all results in %s\n", ARGV[1]); > > }' $LOG_FILE > > > > And use it like this: > > > > LOG_REPORT = $(CURDIR)/report.sh > > > > I plan to renew this patchset, which one of the above methods do you > prefer? IFF it needs to be done I prefer the macro in the Makefile to avoid depending on external scripts that are useless outside of the makefile. BUT, my point remains that I adopted this so that I could quickly and visually check that everything was OK. I'm fine with any other method but I do not want to have to carefully read all these lines to make sure I'm not mixing a "8" with a "0" (I'm mentioning this one because it's exactly the one I had when I decided to add the extra values). For example if you prepend "FAILURE: ", "WARNING: ", "SUCCESS: " in front of these lines to summarize them depending on the highest level encountered (success, skipped, failed), then I'm fine because it's easy to check that all lines show the same word. > For the always print statement: > > printf(" See all results in %s\n", ARGV[1]); }' Then please put it on its own line without the leading space, this will be even more readable. > I will paste the reason why I need it, as mentioned in [1], if you still > need a clean test report, I will give up this change ;-) No worries, I don't want to be annoying if you need something, but I don't want to be annoyed by changes either :-) thanks, Willy
On Wed, Jun 07, 2023 at 01:52:00PM +0800, Zhangjin Wu wrote: > > Hi, Willy > > > (...) > > > > > > Ok, thanks, what about this? > > > > > > # LOG_REPORT: report the test results > > > LOG_REPORT := awk '/\[OK\][\r]*$$/{p++} /\[FAIL\][\r]*$$/{f++} /\[SKIPPED\][\r]*$$/{s++} \ > > > END{ printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); \ > > > printf(" See all results in %s\n", ARGV[1]); }' > > > > > > run-user: nolibc-test > > > $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : > > > $(Q)$(LOG_REPORT) $(CURDIR)/run.out > > > > > > run: kernel > > > $(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out" > > > $(Q)$(LOG_REPORT) $(CURDIR)/run.out > > > > > > rerun: > > > $(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out" > > > $(Q)$(LOG_REPORT) $(CURDIR)/run.out > > > > > > Or we directly add a standalone test report script? something like > > > tools/testing/selftests/nolibc/report.sh > > > > > > #!/bin/sh > > > # > > > # report.sh -- report the test results of nolibc-test > > > # > > > > > > LOG_FILE=$1 > > > [ ! -f "$LOG_FILE" ] && echo "Usage: $0 /path/to/run.out" > > > > > > awk ' > > > /\[OK\][\r]*$$/{ p++ } > > > /\[FAIL\][\r]*$$/{ f++ } > > > /\[SKIPPED\][\r]*$$/{ s++ } > > > > > > END { > > > printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); > > > printf(" See all results in %s\n", ARGV[1]); > > > }' $LOG_FILE > > > > > > And use it like this: > > > > > > LOG_REPORT = $(CURDIR)/report.sh > > > > > > > I plan to renew this patchset, which one of the above methods do you > > prefer? > > IFF it needs to be done I prefer the macro in the Makefile to avoid > depending on external scripts that are useless outside of the makefile. > BUT, my point remains that I adopted this so that I could quickly and > visually check that everything was OK. I'm fine with any other method > but I do not want to have to carefully read all these lines to make > sure I'm not mixing a "8" with a "0" (I'm mentioning this one because > it's exactly the one I had when I decided to add the extra values). > For example if you prepend "FAILURE: ", "WARNING: ", "SUCCESS: " in > front of these lines to summarize them depending on the highest level > encountered (success, skipped, failed), then I'm fine because it's > easy to check that all lines show the same word. > Ok. > > For the always print statement: > > > > printf(" See all results in %s\n", ARGV[1]); }' > > Then please put it on its own line without the leading space, this > will be even more readable. > It is a good balance. This may be more useful if we run this from the kselftest framework, seems it is not able to run it from the 'kselftest' target currently, here shares something I have tried: I tried something like this, seems run-user works, but defconfig and run not. diff --git a/tools/testing/selftests/nolibc/Makefile b/tools/testing/selftests/nolibc/Makefile index 4a3a105e1fdf..70693f6501c6 100644 --- a/tools/testing/selftests/nolibc/Makefile +++ b/tools/testing/selftests/nolibc/Makefile @@ -83,6 +83,8 @@ CFLAGS ?= -Os -fno-ident -fno-asynchronous-unwind-tables -std=c89 \ $(CFLAGS_$(ARCH)) $(CFLAGS_STACKPROTECTOR) LDFLAGS := -s +NOLIBC_SUBTARGETS ?= run-user + help: @echo "Supported targets under selftests/nolibc:" @echo " all call the \"run\" target below" @@ -110,7 +112,9 @@ help: @echo " IMAGE_NAME = $(if $(IMAGE_NAME),$(IMAGE_NAME),UNKNOWN_ARCH) [determined from \$$ARCH]" @echo "" -all: run +run_tests: $(NOLIBC_SUBTARGETS) + +all: $(NOLIBC_SUBTARGETS) This can be triggered from the top-level kselftest framework: $ make -C /path/to/linux-stable kselftest TARGETS=nolibc NOLIBC_SUBTARGETS=run-user And seems we still not support O= currently either. > > I will paste the reason why I need it, as mentioned in [1], if you still > > need a clean test report, I will give up this change ;-) > > No worries, I don't want to be annoying if you need something, but I > don't want to be annoyed by changes either :-) > Thanks, Zhangjin > thanks, > Willy
diff --git a/tools/testing/selftests/nolibc/Makefile b/tools/testing/selftests/nolibc/Makefile index be4159837494..8149ace2938a 100644 --- a/tools/testing/selftests/nolibc/Makefile +++ b/tools/testing/selftests/nolibc/Makefile @@ -127,14 +127,18 @@ nolibc-test: nolibc-test.c sysroot/$(ARCH)/include libc-test: nolibc-test.c $(QUIET_CC)$(CC) -o $@ $< -# qemu user-land test -run-user: nolibc-test - $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : +test-report: $(Q)awk '/\[OK\]$$/{p++} /\[FAIL\]$$/{f++} /\[SKIPPED\]$$/{s++} \ END{ printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); \ if (s+f > 0) printf(" See all results in %s\n", ARGV[1]); else print; }' \ $(CURDIR)/run.out +# qemu user-land test +_run-user: nolibc-test + $(Q)qemu-$(QEMU_ARCH) ./nolibc-test > "$(CURDIR)/run.out" || : + +run-user: _run-user test-report + initramfs: nolibc-test $(QUIET_MKDIR)mkdir -p initramfs $(call QUIET_INSTALL, initramfs/init) @@ -147,20 +151,16 @@ kernel: initramfs $(Q)$(MAKE) -C $(srctree) ARCH=$(ARCH) CC=$(CC) CROSS_COMPILE=$(CROSS_COMPILE) $(IMAGE_NAME) CONFIG_INITRAMFS_SOURCE=$(CURDIR)/initramfs # run the tests after building the kernel -run: kernel +_run: kernel $(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out" - $(Q)awk '/\[OK\]$$/{p++} /\[FAIL\]$$/{f++} /\[SKIPPED\]$$/{s++} \ - END{ printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); \ - if (s+f > 0) printf(" See all results in %s\n", ARGV[1]); else print; }' \ - $(CURDIR)/run.out + +run: _run test-report # re-run the tests from an existing kernel -rerun: +_rerun: $(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out" - $(Q)awk '/\[OK\]$$/{p++} /\[FAIL\]$$/{f++} /\[SKIPPED\]$$/{s++} \ - END{ printf("%d test(s) passed, %d skipped, %d failed.", p, s, f); \ - if (s+f > 0) printf(" See all results in %s\n", ARGV[1]); else print; }' \ - $(CURDIR)/run.out + +rerun: _rerun test-report clean: $(call QUIET_CLEAN, sysroot)
A standalone test-report target is added to let the run, run-user and rerun targets share them. Signed-off-by: Zhangjin Wu <falcon@tinylab.org> --- tools/testing/selftests/nolibc/Makefile | 26 ++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-)