From patchwork Sat Aug 8 01:16:50 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Brendan Higgins X-Patchwork-Id: 11706217 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 4D49713B6 for ; Sat, 8 Aug 2020 01:17:18 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id 2651922CBB for ; Sat, 8 Aug 2020 01:17:18 +0000 (UTC) Authentication-Results: mail.kernel.org; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b="fjeWWPzv" Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726166AbgHHBRR (ORCPT ); Fri, 7 Aug 2020 21:17:17 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:60128 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726066AbgHHBRR (ORCPT ); Fri, 7 Aug 2020 21:17:17 -0400 Received: from mail-yb1-xb4a.google.com (mail-yb1-xb4a.google.com [IPv6:2607:f8b0:4864:20::b4a]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id F3CA7C061756 for ; Fri, 7 Aug 2020 18:17:16 -0700 (PDT) Received: by mail-yb1-xb4a.google.com with SMTP id 83so5067461ybf.2 for ; Fri, 07 Aug 2020 18:17:16 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20161025; h=date:message-id:mime-version:subject:from:to:cc; bh=qDtgVx5yXEzQ++OacfHgx9dKEkbfCyUB5rxuAZU49e8=; b=fjeWWPzv3ENtmT1A5ELAglDHmhrNm2w8zH41/GIdI0PkRkqGvjZlzZXH8IBXXBh+uq qKLfdMnIoB8Z4Rd9jMog3//UBJF8IkTSmfoBvcjWijMcDYhquUsxI7xbEdMSQpYv2Fpd oN37Vy7OsQe3BL32UCedj2LRnH3OElFD3Fd6tW/WPn9bx0C4MIv0BuCZDy52nQUMHT6x ZjJ4Z47lFRWOv2h+nxXer+Ffc2mah8yhxxGYKzFOF3IzyVbr5sADj/DhdbpZaowV8NG4 mvY/1bM48GAzG24++yyx3LIFuCLh9PEs5JNuHRMPTiCPlDbFpHe9B9MRgsR+urql5V9/ 71WA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:date:message-id:mime-version:subject:from:to:cc; bh=qDtgVx5yXEzQ++OacfHgx9dKEkbfCyUB5rxuAZU49e8=; b=fRwOyiwd1j5wh5inKYxequ8aV1/LqjetfFZp2d2MZUNROVLB1iRIybhooSY8eh13H+ Tb0v5sGkc/VcU1g3iM1t0fhCcdEbhr3wNlIcQJmhoap82NYRFA+LOy49Q8mAvV/iiK25 zdTv7oCNcEEmmd6hB0tZtw3y/I2yWfKYo1vJmDLCViSv/WiKf9u12GzESqr6/h32kouY iUoJccedlWcGW24GF8R10C/xpb2mKr5d1neGAYGzI+TL6DgtMJ/ls1plIobsP30wJV2o +MSrBKZ7XWEwAKA9tHhu0yhpqtFYiX90Sjxufc1SeNfQ1wbc200Mn3WV0+qPt673eu9C C1fQ== X-Gm-Message-State: AOAM532TZm+mgc+ET3tJ0xGL18Js2u8AJRyGnW9Cz7ZiSSJqajOrWFzW eFfJCT5nTfDn+egyrNodKr2Rr+xXWELhtgfRVoIr3Q== X-Google-Smtp-Source: ABdhPJzCGS/d0450sHolXKm5LfyKAM0CNPtSWf1OCtlSdIckuPs7Hywjaff7wvkUsImsX50dti5npceBSJjkrypq+F0rSQ== X-Received: by 2002:a25:5808:: with SMTP id m8mr25093531ybb.246.1596849436001; Fri, 07 Aug 2020 18:17:16 -0700 (PDT) Date: Fri, 7 Aug 2020 18:16:50 -0700 Message-Id: <20200808011651.2113930-1-brendanhiggins@google.com> Mime-Version: 1.0 X-Mailer: git-send-email 2.28.0.236.gb10cc79966-goog Subject: [PATCH v1 1/2] kunit: tool: fix running kunit_tool from outside kernel tree From: Brendan Higgins To: shuah@kernel.org, davidgow@google.com Cc: linux-kselftest@vger.kernel.org, kunit-dev@googlegroups.com, linux-kernel@vger.kernel.org, Brendan Higgins Sender: linux-kselftest-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kselftest@vger.kernel.org Currently kunit_tool does not work correctly when executed from a path outside of the kernel tree, so make sure that the current working directory is correct and the kunit_dir is properly initialized before running. Signed-off-by: Brendan Higgins --- tools/testing/kunit/kunit.py | 8 ++++++++ 1 file changed, 8 insertions(+) base-commit: 30185b69a2d533c4ba6ca926b8390ce7de495e29 diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 425ef40067e7..96344a11ff1f 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -237,9 +237,14 @@ def main(argv, linux=None): cli_args = parser.parse_args(argv) + if get_kernel_root_path(): + print('cd ' + get_kernel_root_path()) + os.chdir(get_kernel_root_path()) + if cli_args.subcommand == 'run': if not os.path.exists(cli_args.build_dir): os.mkdir(cli_args.build_dir) + create_default_kunitconfig() if not linux: linux = kunit_kernel.LinuxSourceTree() @@ -257,6 +262,7 @@ def main(argv, linux=None): if cli_args.build_dir: if not os.path.exists(cli_args.build_dir): os.mkdir(cli_args.build_dir) + create_default_kunitconfig() if not linux: linux = kunit_kernel.LinuxSourceTree() @@ -273,6 +279,7 @@ def main(argv, linux=None): if cli_args.build_dir: if not os.path.exists(cli_args.build_dir): os.mkdir(cli_args.build_dir) + create_default_kunitconfig() if not linux: linux = kunit_kernel.LinuxSourceTree() @@ -291,6 +298,7 @@ def main(argv, linux=None): if cli_args.build_dir: if not os.path.exists(cli_args.build_dir): os.mkdir(cli_args.build_dir) + create_default_kunitconfig() if not linux: linux = kunit_kernel.LinuxSourceTree() From patchwork Sat Aug 8 01:16:51 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Brendan Higgins X-Patchwork-Id: 11706219 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 1515914B7 for ; Sat, 8 Aug 2020 01:17:42 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id E6D7222CBB for ; Sat, 8 Aug 2020 01:17:41 +0000 (UTC) Authentication-Results: mail.kernel.org; dkim=pass (2048-bit key) header.d=google.com header.i=@google.com header.b="TuoU7TfC" Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726233AbgHHBRl (ORCPT ); Fri, 7 Aug 2020 21:17:41 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:60192 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726159AbgHHBRl (ORCPT ); Fri, 7 Aug 2020 21:17:41 -0400 Received: from mail-qv1-xf49.google.com (mail-qv1-xf49.google.com [IPv6:2607:f8b0:4864:20::f49]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 258F7C061756 for ; Fri, 7 Aug 2020 18:17:41 -0700 (PDT) Received: by mail-qv1-xf49.google.com with SMTP id h6so2767491qvz.14 for ; Fri, 07 Aug 2020 18:17:41 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20161025; h=date:in-reply-to:message-id:mime-version:references:subject:from:to :cc; bh=yd6TGRc1GXqN/EjtG+8Verq36FPJdYPHa8QPD7rKsBw=; b=TuoU7TfCVBfpE2cCm9NmCqDraN4zNzlW7gPwmIpx6K/oM4Q/HzkbI3h82zn0k0TRbb qkjfxDDVmZgwieq5MsHrot7tkDiIegJvaQ81Y8kNk2Nmo/gO3R+by89MaLPEEhLjC2hf iUe0qc2wDryiDiX6lHZeRcMu2C14FwodJ0shPV18YQyjaXg6MPbsf2zCRbO7S8aJIW5U uvlCoPUUOKjEA8Fmdgkwaku1Pbz23uA3WOUL8nRB2PIbWriUNaAcVl5rjPAAB9be/1Hb +b9ixQDQvWydkEdUL+rtl5JHrqC3vBPmND3vMbAsRiD503ib7L5LJVm1OArdlAt8+5K6 1Yqw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:date:in-reply-to:message-id:mime-version :references:subject:from:to:cc; bh=yd6TGRc1GXqN/EjtG+8Verq36FPJdYPHa8QPD7rKsBw=; b=VwSSEMLlENaknX2f1cFe+35VhzkXnsy4IwXZrEpK8UWzb4eznxVcQFgafEjlJLDFJz DSrdxb9yOv5X9+Q6jDYUGAxOXnhw6ad7AXFbP8+sSZSWygRja5owtEFhTt6uyKHx/Tfs rIZOcAIQFRiE/wD08KhjLYwdWvd+Y5U/TW8VAv0ZpAfKzQ4qMZ/gy/8Zg2dNsth+I6GS 1uk712AqSigqBcLVmnUQW4zRg4P6PAFgZ3TkTf32pAz0/klSoZHAys9FzBMkufMmGBRs CuJycIZpE8GreeJ5Ccmo41P+vZr72nYxGhD6g0qItzjqwwwzR71OsIYrMrP5yhZgSFC8 /LcQ== X-Gm-Message-State: AOAM530JEnWBqLpUDxjQ7Y5MKqWZhFDQtLUgbAPxJYIi0z4FuVruq8l8 kY1hUDSBiL8CVnWbdPDs237n6MnQJ3sGGkEk6yLq3w== X-Google-Smtp-Source: ABdhPJx+t440pJAV1URxY4FC4xXu7li3tPKgxdxjbCWQkD0lFIKWnmZ0ha7fd/s17cbCixhZFQD4B9W9K604rY3jrFVlRw== X-Received: by 2002:ad4:44e5:: with SMTP id p5mr17363190qvt.197.1596849460227; Fri, 07 Aug 2020 18:17:40 -0700 (PDT) Date: Fri, 7 Aug 2020 18:16:51 -0700 In-Reply-To: <20200808011651.2113930-1-brendanhiggins@google.com> Message-Id: <20200808011651.2113930-2-brendanhiggins@google.com> Mime-Version: 1.0 References: <20200808011651.2113930-1-brendanhiggins@google.com> X-Mailer: git-send-email 2.28.0.236.gb10cc79966-goog Subject: [PATCH v1 2/2] kunit: tool: allow generating test results in JSON From: Brendan Higgins To: shuah@kernel.org, davidgow@google.com Cc: linux-kselftest@vger.kernel.org, kunit-dev@googlegroups.com, linux-kernel@vger.kernel.org, Heidi Fahim , Brendan Higgins Sender: linux-kselftest-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kselftest@vger.kernel.org From: Heidi Fahim Add a --json flag, which when specified when kunit_tool is run, generates JSON formatted test results conforming to the KernelCI API test_group spec[1]. The user can the new flag to specify a filename as the value to json in order to store the JSON results under linux/. Link[1]: https://api.kernelci.org/schema-test-group.html#post Signed-off-by: Heidi Fahim Signed-off-by: Brendan Higgins --- tools/testing/kunit/kunit.py | 35 +++++++++++--- tools/testing/kunit/kunit_json.py | 63 ++++++++++++++++++++++++++ tools/testing/kunit/kunit_tool_test.py | 33 ++++++++++++++ 3 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 tools/testing/kunit/kunit_json.py diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 96344a11ff1f..485b7c63b967 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -17,6 +17,7 @@ from collections import namedtuple from enum import Enum, auto import kunit_config +import kunit_json import kunit_kernel import kunit_parser @@ -30,9 +31,9 @@ KunitBuildRequest = namedtuple('KunitBuildRequest', KunitExecRequest = namedtuple('KunitExecRequest', ['timeout', 'build_dir', 'alltests']) KunitParseRequest = namedtuple('KunitParseRequest', - ['raw_output', 'input_data']) + ['raw_output', 'input_data', 'build_dir', 'json']) KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', - 'build_dir', 'alltests', + 'build_dir', 'alltests', 'json', 'make_options']) KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] @@ -113,12 +114,22 @@ def parse_tests(request: KunitParseRequest) -> KunitResult: test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, [], 'Tests not Parsed.') + if request.raw_output: kunit_parser.raw_output(request.input_data) else: test_result = kunit_parser.parse_run_tests(request.input_data) parse_end = time.time() + if request.json: + json_obj = kunit_json.get_json_result( + test_result=test_result, + def_config='kunit_defconfig', + build_dir=request.build_dir, + json_path=request.json) + if request.json == 'stdout': + print(json_obj) + if test_result.status != kunit_parser.TestStatus.SUCCESS: return KunitResult(KunitStatus.TEST_FAILURE, test_result, parse_end - parse_start) @@ -151,7 +162,9 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree, return exec_result parse_request = KunitParseRequest(request.raw_output, - exec_result.result) + exec_result.result, + request.build_dir, + request.json) parse_result = parse_tests(parse_request) run_end = time.time() @@ -195,7 +208,12 @@ def add_exec_opts(parser): def add_parse_opts(parser): parser.add_argument('--raw_output', help='don\'t format output from kernel', action='store_true') - + parser.add_argument('--json', + nargs='?', + help='Stores test results in a JSON, and either ' + 'prints to stdout or saves to file if a ' + 'filename is specified', + type=str, const='stdout', default=None) def main(argv, linux=None): parser = argparse.ArgumentParser( @@ -254,6 +272,7 @@ def main(argv, linux=None): cli_args.jobs, cli_args.build_dir, cli_args.alltests, + cli_args.json, cli_args.make_options) result = run_tests(linux, request) if result.status != KunitStatus.SUCCESS: @@ -308,7 +327,9 @@ def main(argv, linux=None): cli_args.alltests) exec_result = exec_tests(linux, exec_request) parse_request = KunitParseRequest(cli_args.raw_output, - exec_result.result) + exec_result.result, + cli_args.build_dir, + cli_args.json) result = parse_tests(parse_request) kunit_parser.print_with_timestamp(( 'Elapsed time: %.3fs\n') % ( @@ -322,7 +343,9 @@ def main(argv, linux=None): with open(cli_args.file, 'r') as f: kunit_output = f.read().splitlines() request = KunitParseRequest(cli_args.raw_output, - kunit_output) + kunit_output, + cli_args.build_dir, + cli_args.json) result = parse_tests(request) if result.status != KunitStatus.SUCCESS: sys.exit(1) diff --git a/tools/testing/kunit/kunit_json.py b/tools/testing/kunit/kunit_json.py new file mode 100644 index 000000000000..624b31b2dbd6 --- /dev/null +++ b/tools/testing/kunit/kunit_json.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Generates JSON from KUnit results according to +# KernelCI spec: https://github.com/kernelci/kernelci-doc/wiki/Test-API +# +# Copyright (C) 2020, Google LLC. +# Author: Heidi Fahim + +import json +import os + +import kunit_parser + +from kunit_parser import TestStatus + +def get_json_result(test_result, def_config, build_dir, json_path): + sub_groups = [] + + # Each test suite is mapped to a KernelCI sub_group + for test_suite in test_result.suites: + sub_group = { + "name": test_suite.name, + "arch": "UM", + "defconfig": def_config, + "build_environment": build_dir, + "test_cases": [], + "lab_name": None, + "kernel": None, + "job": None, + "git_branch": "kselftest", + } + test_cases = [] + # TODO: Add attachments attribute in test_case with detailed + # failure message, see https://api.kernelci.org/schema-test-case.html#get + for case in test_suite.cases: + test_case = {"name": case.name, "status": "FAIL"} + if case.status == TestStatus.SUCCESS: + test_case["status"] = "PASS" + elif case.status == TestStatus.TEST_CRASHED: + test_case["status"] = "ERROR" + test_cases.append(test_case) + sub_group["test_cases"] = test_cases + sub_groups.append(sub_group) + test_group = { + "name": "KUnit Test Group", + "arch": "UM", + "defconfig": def_config, + "build_environment": build_dir, + "sub_groups": sub_groups, + "lab_name": None, + "kernel": None, + "job": None, + "git_branch": "kselftest", + } + json_obj = json.dumps(test_group, indent=4) + if json_path != 'stdout': + with open(json_path, 'w') as result_path: + result_path.write(json_obj) + root = __file__.split('tools/testing/kunit/')[0] + kunit_parser.print_with_timestamp( + "Test results stored in %s" % + os.path.join(root, result_path.name)) + return json_obj diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 287c74d821c3..99c3c5671ea4 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -11,11 +11,13 @@ from unittest import mock import tempfile, shutil # Handling test_tmpdir +import json import os import kunit_config import kunit_parser import kunit_kernel +import kunit_json import kunit test_tmpdir = '' @@ -230,6 +232,37 @@ class KUnitParserTest(unittest.TestCase): result = kunit_parser.parse_run_tests(file.readlines()) self.assertEqual('kunit-resource-test', result.suites[0].name) +class KUnitJsonTest(unittest.TestCase): + + def _json_for(self, log_file): + with(open(get_absolute_path(log_file))) as file: + test_result = kunit_parser.parse_run_tests(file) + json_obj = kunit_json.get_json_result( + test_result=test_result, + def_config='kunit_defconfig', + build_dir=None, + json_path='stdout') + return json.loads(json_obj) + + def test_failed_test_json(self): + result = self._json_for( + 'test_data/test_is_test_passed-failure.log') + self.assertEqual( + {'name': 'example_simple_test', 'status': 'FAIL'}, + result["sub_groups"][1]["test_cases"][0]) + + def test_crashed_test_json(self): + result = self._json_for( + 'test_data/test_is_test_passed-crash.log') + self.assertEqual( + {'name': 'example_simple_test', 'status': 'ERROR'}, + result["sub_groups"][1]["test_cases"][0]) + + def test_no_tests_json(self): + result = self._json_for( + 'test_data/test_is_test_passed-no_tests_run.log') + self.assertEqual(0, len(result['sub_groups'])) + class StrContains(str): def __eq__(self, other): return self in other