From patchwork Tue Jun 25 03:26:31 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Daniel Sangorrin X-Patchwork-Id: 11014577 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 755D4112C for ; Tue, 25 Jun 2019 03:27:08 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 63F4628A8C for ; Tue, 25 Jun 2019 03:27:08 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 53F0E28ACE; Tue, 25 Jun 2019 03:27:08 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from mail.linuxfoundation.org (mail.linuxfoundation.org [140.211.169.12]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id D8F9628A8C for ; Tue, 25 Jun 2019 03:27:07 +0000 (UTC) Received: from mail.linux-foundation.org (localhost [127.0.0.1]) by mail.linuxfoundation.org (Postfix) with ESMTP id EC50BC2A; Tue, 25 Jun 2019 03:26:57 +0000 (UTC) X-Original-To: cip-dev@lists.cip-project.org Delivered-To: cip-dev@mail.linuxfoundation.org Received: from smtp1.linuxfoundation.org (smtp1.linux-foundation.org [172.17.192.35]) by mail.linuxfoundation.org (Postfix) with ESMTPS id DD2B3AF0 for ; Tue, 25 Jun 2019 03:26:56 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from mo-csw.securemx.jp (mo-csw1516.securemx.jp [210.130.202.155]) by smtp1.linuxfoundation.org (Postfix) with ESMTPS id 3536A82C for ; Tue, 25 Jun 2019 03:26:56 +0000 (UTC) Received: by mo-csw.securemx.jp (mx-mo-csw1516) id x5P3Qfpj030665; Tue, 25 Jun 2019 12:26:41 +0900 X-Iguazu-Qid: 34tKu0vdNfDAfab8vU X-Iguazu-QSIG: v=2; s=0; t=1561433201; q=34tKu0vdNfDAfab8vU; m=jf4SbnYz53tenbpUVcgBHZzga28LzHWB4XtO9AnJzDM= Received: from imx12.toshiba.co.jp (imx12.toshiba.co.jp [61.202.160.132]) by relay.securemx.jp (mx-mr1510) id x5P3QeW7023076; Tue, 25 Jun 2019 12:26:41 +0900 Received: from enc02.toshiba.co.jp ([61.202.160.51]) by imx12.toshiba.co.jp with ESMTP id x5P3QeOr029373; Tue, 25 Jun 2019 12:26:40 +0900 (JST) Received: from hop101.toshiba.co.jp ([133.199.85.107]) by enc02.toshiba.co.jp with ESMTP id x5P3QdED002826; Tue, 25 Jun 2019 12:26:40 +0900 From: Daniel Sangorrin To: ben.hutchings@codethink.co.uk Date: Tue, 25 Jun 2019 12:26:31 +0900 X-TSB-HOP: ON Message-Id: <20190625032636.10694-2-daniel.sangorrin@toshiba.co.jp> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190625032636.10694-1-daniel.sangorrin@toshiba.co.jp> References: <20190625032636.10694-1-daniel.sangorrin@toshiba.co.jp> Cc: cip-dev@lists.cip-project.org Subject: [cip-dev] [cip-kernel-sec 1/6] check_git_repo: add checks to the local repository X-BeenThere: cip-dev@lists.cip-project.org X-Mailman-Version: 2.1.12 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Sender: cip-dev-bounces@lists.cip-project.org Errors-To: cip-dev-bounces@lists.cip-project.org X-Virus-Scanned: ClamAV using ClamSMTP Add checks to make sure that the local repository exists and has the configured remotes in place. Signed-off-by: Daniel Sangorrin --- scripts/import_stable.py | 1 + scripts/kernel_sec/branch.py | 21 +++++++++++++++++++++ scripts/report_affected.py | 1 + scripts/webview.py | 1 + 4 files changed, 24 insertions(+) diff --git a/scripts/import_stable.py b/scripts/import_stable.py index 2c88b07..26fcc2f 100755 --- a/scripts/import_stable.py +++ b/scripts/import_stable.py @@ -190,4 +190,5 @@ if __name__ == '__main__': remotes = kernel_sec.branch.get_remotes(args.remote_name, mainline=args.mainline_remote_name, stable=args.stable_remote_name) + kernel_sec.branch.check_git_repo(args.git_repo, remotes) main(args.git_repo, remotes, args.debug) diff --git a/scripts/kernel_sec/branch.py b/scripts/kernel_sec/branch.py index 10f3339..a138b96 100644 --- a/scripts/kernel_sec/branch.py +++ b/scripts/kernel_sec/branch.py @@ -4,10 +4,12 @@ # Public License, Version 3 or later. See http://www.gnu.org/copyleft/gpl.html # for details. +import argparse import io import os import re import subprocess +import sys import time import urllib.error import urllib.request @@ -219,3 +221,22 @@ def get_remotes(mappings, mainline=None, stable=None): if stable: remotes['stable']['git_name'] = stable return remotes + + +def check_git_repo(git_repo, remotes): + if not os.path.isdir(git_repo): + msg = "directory %r not present" % git_repo + raise argparse.ArgumentError(None, msg) + # .git could be a regular file (worktrees) or a directory + if not os.path.exists(os.path.join(git_repo, '.git')): + msg = "directory %r is not a git repository" % git_repo + raise argparse.ArgumentError(None, msg) + + current_remotes = subprocess.check_output( + ['git', 'remote', 'show'], cwd=git_repo).decode( + sys.stdout.encoding).strip().split('\n') + for key in remotes.keys(): + remote = remotes[key] # __getitem__ will add git_name + if remote['git_name'] not in current_remotes: + msg = "remote %r not in git repository" % remote['git_name'] + raise argparse.ArgumentError(None, msg) diff --git a/scripts/report_affected.py b/scripts/report_affected.py index 879c021..d2f1f22 100755 --- a/scripts/report_affected.py +++ b/scripts/report_affected.py @@ -100,5 +100,6 @@ if __name__ == '__main__': remotes = kernel_sec.branch.get_remotes(args.remote_name, mainline=args.mainline_remote_name, stable=args.stable_remote_name) + kernel_sec.branch.check_git_repo(args.git_repo, remotes) main(args.git_repo, remotes, args.only_fixed_upstream, args.include_ignored, *args.branches) diff --git a/scripts/webview.py b/scripts/webview.py index a3a643b..52a7b15 100755 --- a/scripts/webview.py +++ b/scripts/webview.py @@ -219,6 +219,7 @@ if __name__ == '__main__': remotes = kernel_sec.branch.get_remotes(args.remote_name, mainline=args.mainline_remote_name, stable=args.stable_remote_name) + kernel_sec.branch.check_git_repo(args.git_repo, remotes) conf = { '/static/style.css': { From patchwork Tue Jun 25 03:26:32 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Daniel Sangorrin X-Patchwork-Id: 11014583 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id B542313AF for ; Tue, 25 Jun 2019 03:27:22 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id A60EA28A8C for ; Tue, 25 Jun 2019 03:27:22 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 9A6F128ACE; Tue, 25 Jun 2019 03:27:22 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from mail.linuxfoundation.org (mail.linuxfoundation.org [140.211.169.12]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id 1985228A8C for ; Tue, 25 Jun 2019 03:27:22 +0000 (UTC) Received: from mail.linux-foundation.org (localhost [127.0.0.1]) by mail.linuxfoundation.org (Postfix) with ESMTP id 3BB64B7D; Tue, 25 Jun 2019 03:26:58 +0000 (UTC) X-Original-To: cip-dev@lists.cip-project.org Delivered-To: cip-dev@mail.linuxfoundation.org Received: from smtp1.linuxfoundation.org (smtp1.linux-foundation.org [172.17.192.35]) by mail.linuxfoundation.org (Postfix) with ESMTPS id 91327AE0 for ; Tue, 25 Jun 2019 03:26:57 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from mo-csw.securemx.jp (mo-csw1115.securemx.jp [210.130.202.157]) by smtp1.linuxfoundation.org (Postfix) with ESMTPS id 926C87FD for ; Tue, 25 Jun 2019 03:26:56 +0000 (UTC) Received: by mo-csw.securemx.jp (mx-mo-csw1115) id x5P3QfU1021456; Tue, 25 Jun 2019 12:26:41 +0900 X-Iguazu-Qid: 2wGrCre3N0ykLMqXWU X-Iguazu-QSIG: v=2; s=0; t=1561433201; q=2wGrCre3N0ykLMqXWU; m=pgXKsMFWFQZZLXeEiVJd+oBcSDPldWdYetD5FcnRiFs= Received: from imx2.toshiba.co.jp (imx2.toshiba.co.jp [106.186.93.51]) by relay.securemx.jp (mx-mr1112) id x5P3Qerx017098; Tue, 25 Jun 2019 12:26:41 +0900 Received: from enc01.localdomain ([106.186.93.100]) by imx2.toshiba.co.jp with ESMTP id x5P3Qenh022260; Tue, 25 Jun 2019 12:26:40 +0900 (JST) Received: from hop001.toshiba.co.jp ([133.199.164.63]) by enc01.localdomain with ESMTP id x5P3QdLb026418; Tue, 25 Jun 2019 12:26:40 +0900 From: Daniel Sangorrin To: ben.hutchings@codethink.co.uk Date: Tue, 25 Jun 2019 12:26:32 +0900 X-TSB-HOP: ON Message-Id: <20190625032636.10694-3-daniel.sangorrin@toshiba.co.jp> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190625032636.10694-1-daniel.sangorrin@toshiba.co.jp> References: <20190625032636.10694-1-daniel.sangorrin@toshiba.co.jp> Cc: cip-dev@lists.cip-project.org Subject: [cip-dev] [cip-kernel-sec 2/6] prepare_remotes: helper script to prepare local repo X-BeenThere: cip-dev@lists.cip-project.org X-Mailman-Version: 2.1.12 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Sender: cip-dev-bounces@lists.cip-project.org Errors-To: cip-dev-bounces@lists.cip-project.org X-Virus-Scanned: ClamAV using ClamSMTP Helper script that prepares the local git repository with the configured remote branches. Expert developers and kernel maintainers will probably have their own worktrees but for new users or a quickstart, this script should be helpful. Signed-off-by: Daniel Sangorrin --- README.md | 5 +++ conf/remotes.yml | 3 ++ scripts/import_stable.py | 8 ++--- scripts/kernel_sec/branch.py | 10 ++++++ scripts/prepare_remotes.py | 62 ++++++++++++++++++++++++++++++++++++ 5 files changed, 82 insertions(+), 6 deletions(-) create mode 100755 scripts/prepare_remotes.py diff --git a/README.md b/README.md index 4c5808f..576cc75 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,10 @@ this is assumed to be in `../kernel`, with remotes configured in stable and cip repositories. These can be overridden by command-line options or configuration (`~/.config/kernel-sec/remotes.yml`). +* `scripts/prepare_remotes.py` - creates the local git repository +and adds all configured remotes. You may prefer to skip this script +and configure the repository by hand. + * `scripts/import_debian.py` - import information from Debian's `kernel_sec` project. It includes all issues that Debian considers active or that are already tracked here. @@ -81,6 +85,7 @@ with the keys: branch from this remote. * `git_name`: (optional) The name actually used for this git remote, if it's different from the default. +* `git_repo_url`: URL of the remote git repository. ## Contributions diff --git a/conf/remotes.yml b/conf/remotes.yml index 51c523d..cfaa35f 100644 --- a/conf/remotes.yml +++ b/conf/remotes.yml @@ -1,6 +1,9 @@ torvalds: commit_url_prefix: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id= + git_repo_url: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git stable: commit_url_prefix: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git/commit?id= + git_repo_url: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git cip: commit_url_prefix: https://git.kernel.org/pub/scm/linux/kernel/git/cip/linux-cip.git/commit?id= + git_repo_url: https://git.kernel.org/pub/scm/linux/kernel/git/cip/linux-cip.git diff --git a/scripts/import_stable.py b/scripts/import_stable.py index 26fcc2f..194219d 100755 --- a/scripts/import_stable.py +++ b/scripts/import_stable.py @@ -31,11 +31,6 @@ BACKPORT_COMMIT_BOTTOM_RE = re.compile( .format(**RE_USE)) -def update(git_repo, remote_name): - subprocess.check_call(['git', 'remote', 'update', remote_name], - cwd=git_repo) - - def get_backports(git_repo, remotes, branches, debug=False): backports = {} @@ -140,7 +135,8 @@ def main(git_repo, remotes, debug=False): remote_names = set(branch['git_remote'] for branch in branches) for remote_name in remote_names: - update(git_repo, remotes[remote_name]['git_name']) + kernel_sec.branch.remote_update( + git_repo, remotes[remote_name]['git_name']) backports = get_backports(git_repo, remotes, branches, debug) c_b_map = kernel_sec.branch.CommitBranchMap(git_repo, remotes, branches) diff --git a/scripts/kernel_sec/branch.py b/scripts/kernel_sec/branch.py index a138b96..0023497 100644 --- a/scripts/kernel_sec/branch.py +++ b/scripts/kernel_sec/branch.py @@ -223,6 +223,16 @@ def get_remotes(mappings, mainline=None, stable=None): return remotes +def remote_update(git_repo, remote_name): + subprocess.check_call(['git', 'remote', 'update', remote_name], + cwd=git_repo) + + +def remote_add(git_repo, remote_name, remote_url): + subprocess.check_call(['git', 'remote', 'add', remote_name, remote_url], + cwd=git_repo) + + def check_git_repo(git_repo, remotes): if not os.path.isdir(git_repo): msg = "directory %r not present" % git_repo diff --git a/scripts/prepare_remotes.py b/scripts/prepare_remotes.py new file mode 100755 index 0000000..59310fc --- /dev/null +++ b/scripts/prepare_remotes.py @@ -0,0 +1,62 @@ +#!/usr/bin/python3 + +# Copyright 2019 Toshiba corp. +# Based on import_stable.py by Codethink Ltd. +# +# This script is distributed under the terms and conditions of the GNU General +# Public License, Version 3 or later. See http://www.gnu.org/copyleft/gpl.html +# for details. + +# Helper script that prepares the local git repository with the configured +# remote branches + +import argparse +import os +import subprocess + +import kernel_sec.branch + + +def main(git_repo, remotes): + if os.path.isdir(git_repo): + msg = "directory %r already exists" % git_repo + raise argparse.ArgumentError(None, msg) + else: + os.mkdir(git_repo) + subprocess.check_call(['git', 'init', '.'], cwd=git_repo) + + for key in remotes.keys(): + remote = remotes[key] # __getitem__ will add git_name + kernel_sec.branch.remote_add( + git_repo, remote['git_name'], remote['git_repo_url']) + kernel_sec.branch.remote_update(git_repo, remote['git_name']) + + # self-check + kernel_sec.branch.check_git_repo(git_repo, remotes) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=('Prepare local git repository with configured remotes.')) + parser.add_argument('--git-repo', + dest='git_repo', default='../kernel', + help=('local git repository location ' + '(default: ../kernel)'), + metavar='DIRECTORY') + parser.add_argument('--remote-name', + dest='remote_name', action='append', default=[], + help='git remote name mappings, e.g. stable:korg-stable', + metavar='NAME:OTHER-NAME') + parser.add_argument('--mainline-remote', + dest='mainline_remote_name', + help="git remote name to use instead of 'torvalds'", + metavar='OTHER-NAME') + parser.add_argument('--stable-remote', + dest='stable_remote_name', + help="git remote name to use instead of 'stable'", + metavar='OTHER-NAME') + args = parser.parse_args() + remotes = kernel_sec.branch.get_remotes(args.remote_name, + mainline=args.mainline_remote_name, + stable=args.stable_remote_name) + main(args.git_repo, remotes) From patchwork Tue Jun 25 03:26:33 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Daniel Sangorrin X-Patchwork-Id: 11014581 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 92409112C for ; Tue, 25 Jun 2019 03:27:17 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 82E3128A8C for ; Tue, 25 Jun 2019 03:27:17 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 773DD28ACE; Tue, 25 Jun 2019 03:27:17 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from mail.linuxfoundation.org (mail.linuxfoundation.org [140.211.169.12]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id 2D48528A8C for ; Tue, 25 Jun 2019 03:27:17 +0000 (UTC) Received: from mail.linux-foundation.org (localhost [127.0.0.1]) by mail.linuxfoundation.org (Postfix) with ESMTP id 25738B6D; Tue, 25 Jun 2019 03:26:58 +0000 (UTC) X-Original-To: cip-dev@lists.cip-project.org Delivered-To: cip-dev@mail.linuxfoundation.org Received: from smtp1.linuxfoundation.org (smtp1.linux-foundation.org [172.17.192.35]) by mail.linuxfoundation.org (Postfix) with ESMTPS id 5D30AAE0 for ; Tue, 25 Jun 2019 03:26:57 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from mo-csw.securemx.jp (mo-csw1116.securemx.jp [210.130.202.158]) by smtp1.linuxfoundation.org (Postfix) with ESMTPS id ABBCE831 for ; Tue, 25 Jun 2019 03:26:56 +0000 (UTC) Received: by mo-csw.securemx.jp (mx-mo-csw1116) id x5P3Qfxm017075; Tue, 25 Jun 2019 12:26:41 +0900 X-Iguazu-Qid: 2wGrbTCrMxwtmvrMzw X-Iguazu-QSIG: v=2; s=0; t=1561433201; q=2wGrbTCrMxwtmvrMzw; m=AsL1dKWh7Avua0fOGAamhffPm9XBsbJH+uRluOTsqR4= Received: from imx2.toshiba.co.jp (imx2.toshiba.co.jp [106.186.93.51]) by relay.securemx.jp (mx-mr1110) id x5P3QeZY030308; Tue, 25 Jun 2019 12:26:41 +0900 Received: from enc01.localdomain ([106.186.93.100]) by imx2.toshiba.co.jp with ESMTP id x5P3Qeox022263; Tue, 25 Jun 2019 12:26:40 +0900 (JST) Received: from hop001.toshiba.co.jp ([133.199.164.63]) by enc01.localdomain with ESMTP id x5P3QeWH026422; Tue, 25 Jun 2019 12:26:40 +0900 From: Daniel Sangorrin To: ben.hutchings@codethink.co.uk Date: Tue, 25 Jun 2019 12:26:33 +0900 X-TSB-HOP: ON Message-Id: <20190625032636.10694-4-daniel.sangorrin@toshiba.co.jp> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190625032636.10694-1-daniel.sangorrin@toshiba.co.jp> References: <20190625032636.10694-1-daniel.sangorrin@toshiba.co.jp> Cc: cip-dev@lists.cip-project.org Subject: [cip-dev] [cip-kernel-sec 3/6] report_affected: fix code when branches are specified X-BeenThere: cip-dev@lists.cip-project.org X-Mailman-Version: 2.1.12 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Sender: cip-dev-bounces@lists.cip-project.org Errors-To: cip-dev-bounces@lists.cip-project.org X-Virus-Scanned: ClamAV using ClamSMTP The previous code could not handle branches with names other than stable branch names. For example, passing "linux-4.4.y-cip" as a branch would return an error. Signed-off-by: Daniel Sangorrin --- scripts/kernel_sec/branch.py | 8 -------- scripts/report_affected.py | 15 +++++++++------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/scripts/kernel_sec/branch.py b/scripts/kernel_sec/branch.py index 0023497..ef88b54 100644 --- a/scripts/kernel_sec/branch.py +++ b/scripts/kernel_sec/branch.py @@ -21,9 +21,6 @@ import yaml from . import version -_STABLE_BRANCH_RE = re.compile(r'^linux-([\d.]+)\.y$') - - def get_base_ver_stable_branch(base_ver): branch_name = 'linux-%s.y' % base_ver return { @@ -34,11 +31,6 @@ def get_base_ver_stable_branch(base_ver): } -def get_stable_branch(branch_name): - match = _STABLE_BRANCH_RE.match(branch_name) - return match and get_base_ver_stable_branch(match.group(1)) - - def _extract_live_stable_branches(doc): xhtml_ns = 'http://www.w3.org/1999/xhtml' ns = {'html': xhtml_ns} diff --git a/scripts/report_affected.py b/scripts/report_affected.py index d2f1f22..bd22e29 100755 --- a/scripts/report_affected.py +++ b/scripts/report_affected.py @@ -18,14 +18,17 @@ import kernel_sec.version def main(git_repo, remotes, only_fixed_upstream, include_ignored, *branch_names): + live_branches = kernel_sec.branch.get_live_branches() if branch_names: - # Support stable release strings as shorthand for stable branches - branches = [kernel_sec.branch.get_base_ver_stable_branch(name) - if name[0].isdigit() - else kernel_sec.branch.get_stable_branch(name) - for name in branch_names] + branches = [] + for branch in live_branches: + for name in branch_names: + if name[0].isdigit(): + name = 'linux-%s.y' % name + if branch['short_name'] == name: + branches.append(branch) else: - branches = kernel_sec.branch.get_live_branches() + branches = live_branches if only_fixed_upstream: branches = [branch for branch in branches if branch['short_name'] != 'mainline'] From patchwork Tue Jun 25 03:26:34 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Daniel Sangorrin X-Patchwork-Id: 11014575 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 7449B13AF for ; Tue, 25 Jun 2019 03:27:01 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 60B0828B75 for ; Tue, 25 Jun 2019 03:27:01 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 51D8928B91; Tue, 25 Jun 2019 03:27:01 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from mail.linuxfoundation.org (mail.linuxfoundation.org [140.211.169.12]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id D6DBB28B70 for ; Tue, 25 Jun 2019 03:27:00 +0000 (UTC) Received: from mail.linux-foundation.org (localhost [127.0.0.1]) by mail.linuxfoundation.org (Postfix) with ESMTP id CD116AF0; Tue, 25 Jun 2019 03:26:57 +0000 (UTC) X-Original-To: cip-dev@lists.cip-project.org Delivered-To: cip-dev@mail.linuxfoundation.org Received: from smtp1.linuxfoundation.org (smtp1.linux-foundation.org [172.17.192.35]) by mail.linuxfoundation.org (Postfix) with ESMTPS id 82702AB5 for ; Tue, 25 Jun 2019 03:26:56 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from mo-csw.securemx.jp (mo-csw1515.securemx.jp [210.130.202.154]) by smtp1.linuxfoundation.org (Postfix) with ESMTPS id 1244E7FD for ; Tue, 25 Jun 2019 03:26:55 +0000 (UTC) Received: by mo-csw.securemx.jp (mx-mo-csw1515) id x5P3QfhX010456; Tue, 25 Jun 2019 12:26:41 +0900 X-Iguazu-Qid: 34tMRO62Mdvy29wRZU X-Iguazu-QSIG: v=2; s=0; t=1561433201; q=34tMRO62Mdvy29wRZU; m=xOYW5xLWTSniCCu2SEJ509LQWPB9fl7LonFenda4TqI= Received: from imx12.toshiba.co.jp (imx12.toshiba.co.jp [61.202.160.132]) by relay.securemx.jp (mx-mr1512) id x5P3Qe8d000444; Tue, 25 Jun 2019 12:26:41 +0900 Received: from enc02.toshiba.co.jp ([61.202.160.51]) by imx12.toshiba.co.jp with ESMTP id x5P3Qet8029377; Tue, 25 Jun 2019 12:26:40 +0900 (JST) Received: from hop101.toshiba.co.jp ([133.199.85.107]) by enc02.toshiba.co.jp with ESMTP id x5P3QeuB002831; Tue, 25 Jun 2019 12:26:40 +0900 From: Daniel Sangorrin To: ben.hutchings@codethink.co.uk Date: Tue, 25 Jun 2019 12:26:34 +0900 X-TSB-HOP: ON Message-Id: <20190625032636.10694-5-daniel.sangorrin@toshiba.co.jp> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190625032636.10694-1-daniel.sangorrin@toshiba.co.jp> References: <20190625032636.10694-1-daniel.sangorrin@toshiba.co.jp> Cc: cip-dev@lists.cip-project.org Subject: [cip-dev] [cip-kernel-sec 4/6] report_affected: check user supplied branch names X-BeenThere: cip-dev@lists.cip-project.org X-Mailman-Version: 2.1.12 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Sender: cip-dev-bounces@lists.cip-project.org Errors-To: cip-dev-bounces@lists.cip-project.org X-Virus-Scanned: ClamAV using ClamSMTP This makes sure that we return when the user supplied branch names are not within the configured branches. Signed-off-by: Daniel Sangorrin --- scripts/report_affected.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/report_affected.py b/scripts/report_affected.py index bd22e29..7557dc8 100755 --- a/scripts/report_affected.py +++ b/scripts/report_affected.py @@ -27,6 +27,9 @@ def main(git_repo, remotes, name = 'linux-%s.y' % name if branch['short_name'] == name: branches.append(branch) + if not branches: + msg = "supplied branches didn't match any known branch" + raise argparse.ArgumentError(None, msg) else: branches = live_branches if only_fixed_upstream: From patchwork Tue Jun 25 03:26:35 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Daniel Sangorrin X-Patchwork-Id: 11014573 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id E5B9B14E5 for ; Tue, 25 Jun 2019 03:26:57 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id D673E28A8C for ; Tue, 25 Jun 2019 03:26:57 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id C7CE728ACE; Tue, 25 Jun 2019 03:26:57 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from mail.linuxfoundation.org (mail.linuxfoundation.org [140.211.169.12]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id 230D128ABA for ; Tue, 25 Jun 2019 03:26:56 +0000 (UTC) Received: from mail.linux-foundation.org (localhost [127.0.0.1]) by mail.linuxfoundation.org (Postfix) with ESMTP id A58C0AB5; Tue, 25 Jun 2019 03:26:56 +0000 (UTC) X-Original-To: cip-dev@lists.cip-project.org Delivered-To: cip-dev@mail.linuxfoundation.org Received: from smtp1.linuxfoundation.org (smtp1.linux-foundation.org [172.17.192.35]) by mail.linuxfoundation.org (Postfix) with ESMTPS id F349BAB5 for ; Tue, 25 Jun 2019 03:26:55 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from mo-csw.securemx.jp (mo-csw1114.securemx.jp [210.130.202.156]) by smtp1.linuxfoundation.org (Postfix) with ESMTPS id 177867FD for ; Tue, 25 Jun 2019 03:26:54 +0000 (UTC) Received: by mo-csw.securemx.jp (mx-mo-csw1114) id x5P3QgtC010314; Tue, 25 Jun 2019 12:26:42 +0900 X-Iguazu-Qid: 2wGqtQ13rynb9UKsx5 X-Iguazu-QSIG: v=2; s=0; t=1561433201; q=2wGqtQ13rynb9UKsx5; m=4GjZ7cMGDdqAcHZFo4qWcFZDrtExzOdx0yWLJB6pQtw= Received: from imx2.toshiba.co.jp (imx2.toshiba.co.jp [106.186.93.51]) by relay.securemx.jp (mx-mr1111) id x5P3Qfdv001965; Tue, 25 Jun 2019 12:26:41 +0900 Received: from enc01.localdomain ([106.186.93.100]) by imx2.toshiba.co.jp with ESMTP id x5P3Qfvc022269; Tue, 25 Jun 2019 12:26:41 +0900 (JST) Received: from hop001.toshiba.co.jp ([133.199.164.63]) by enc01.localdomain with ESMTP id x5P3Qeh7026432; Tue, 25 Jun 2019 12:26:41 +0900 From: Daniel Sangorrin To: ben.hutchings@codethink.co.uk Date: Tue, 25 Jun 2019 12:26:35 +0900 X-TSB-HOP: ON Message-Id: <20190625032636.10694-6-daniel.sangorrin@toshiba.co.jp> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190625032636.10694-1-daniel.sangorrin@toshiba.co.jp> References: <20190625032636.10694-1-daniel.sangorrin@toshiba.co.jp> Cc: cip-dev@lists.cip-project.org Subject: [cip-dev] [cip-kernel-sec 5/6] report_affected: add support for reporting on tags X-BeenThere: cip-dev@lists.cip-project.org X-Mailman-Version: 2.1.12 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Sender: cip-dev-bounces@lists.cip-project.org Errors-To: cip-dev-bounces@lists.cip-project.org X-Virus-Scanned: ClamAV using ClamSMTP Reporting on tags is useful for product engineers that have shipped a kernel with a specific tag and need to know which issues affect their product after some time. Signed-off-by: Daniel Sangorrin --- scripts/report_affected.py | 60 ++++++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/scripts/report_affected.py b/scripts/report_affected.py index 7557dc8..32e9345 100755 --- a/scripts/report_affected.py +++ b/scripts/report_affected.py @@ -9,7 +9,9 @@ # Report issues affecting each stable branch. import argparse +import copy import subprocess +import re import kernel_sec.branch import kernel_sec.issue @@ -23,10 +25,26 @@ def main(git_repo, remotes, branches = [] for branch in live_branches: for name in branch_names: + # there could be multiple tags for the same branch + branch_copy = copy.deepcopy(branch) + if name[0] == 'v': + # a stable tag, e.g. v4.4.92-cip11 + branch_copy['tag'] = name + match = re.match(r'^v(\d+\.\d+).*', name) + if not match: + raise ValueError('failed to parse tag %r' % name) + if 'cip' in name: + name = 'linux-%s.y-cip' % match.group(1) + else: + name = 'linux-%s.y' % match.group(1) + if '/' in name: + # a possibly custom tag, e.g. product-v1 + branch_copy['tag'] = name.split('/')[1] + name = name.split('/')[0] if name[0].isdigit(): name = 'linux-%s.y' % name - if branch['short_name'] == name: - branches.append(branch) + if branch_copy['short_name'] == name: + branches.append(branch_copy) if not branches: msg = "supplied branches didn't match any known branch" raise argparse.ArgumentError(None, msg) @@ -40,6 +58,18 @@ def main(git_repo, remotes, c_b_map = kernel_sec.branch.CommitBranchMap(git_repo, remotes, branches) + # cache tag commits and set full_name to show the tag + tag_commits = {} + for branch in branches: + if 'tag' in branch: + start = 'v' + branch['base_ver'] + end = branch['tag'] + for commit in kernel_sec.branch._get_commits(git_repo, end, start): + tag_commits.setdefault(end, []).append(commit) + branch['full_name'] = '/'.join([branch['short_name'], end]) + else: + branch['full_name'] = branch['short_name'] + branch_issues = {} issues = set(kernel_sec.issue.get_list()) @@ -60,14 +90,24 @@ def main(git_repo, remotes, if not include_ignored and ignore.get(branch_name): continue + # Check if the branch is affected. If not and the issue was fixed + # on that branch, then make sure the tag contains that fix if kernel_sec.issue.affects_branch( issue, branch, c_b_map.is_commit_in_branch): - branch_issues.setdefault(branch_name, []).append(cve_id) + branch_issues.setdefault( + branch['full_name'], []).append(cve_id) + elif 'tag' in branch and fixed: + if fixed.get(branch_name, 'never') == 'never': + continue + for commit in fixed[branch_name]: + if commit not in tag_commits[branch['tag']]: + branch_issues.setdefault( + branch['full_name'], []).append(cve_id) + break for branch in branches: - branch_name = branch['short_name'] - print('%s:' % branch_name, - *sorted(branch_issues.get(branch_name, []), + print('%s:' % branch['full_name'], + *sorted(branch_issues.get(branch['full_name'], []), key=kernel_sec.issue.get_id_sort_key)) @@ -99,9 +139,11 @@ if __name__ == '__main__': help='include issues that have been marked as ignored') parser.add_argument('branches', nargs='*', - help=('specific branch to report on ' - '(default: all active branches)'), - metavar='BRANCH') + help=('specific branch[/tag] or stable tag to ' + 'report on (default: all active branches). ' + 'e.g. linux-4.14.y linux-4.4.y/v4.4.107 ' + 'v4.4.181-cip33 linux-4.19.y-cip/myproduct-v33'), + metavar='[BRANCH[/TAG]|TAG]') args = parser.parse_args() remotes = kernel_sec.branch.get_remotes(args.remote_name, mainline=args.mainline_remote_name, From patchwork Tue Jun 25 03:26:36 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Daniel Sangorrin X-Patchwork-Id: 11014579 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 13E98112C for ; Tue, 25 Jun 2019 03:27:13 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 0315D28A8C for ; Tue, 25 Jun 2019 03:27:13 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id EA0A028ACE; Tue, 25 Jun 2019 03:27:12 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from mail.linuxfoundation.org (mail.linuxfoundation.org [140.211.169.12]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id 6C34A28A8C for ; Tue, 25 Jun 2019 03:27:12 +0000 (UTC) Received: from mail.linux-foundation.org (localhost [127.0.0.1]) by mail.linuxfoundation.org (Postfix) with ESMTP id 0E428B65; Tue, 25 Jun 2019 03:26:58 +0000 (UTC) X-Original-To: cip-dev@lists.cip-project.org Delivered-To: cip-dev@mail.linuxfoundation.org Received: from smtp1.linuxfoundation.org (smtp1.linux-foundation.org [172.17.192.35]) by mail.linuxfoundation.org (Postfix) with ESMTPS id 43A84AF0 for ; Tue, 25 Jun 2019 03:26:57 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from mo-csw.securemx.jp (mo-csw1115.securemx.jp [210.130.202.157]) by smtp1.linuxfoundation.org (Postfix) with ESMTPS id 7F7ED82F for ; Tue, 25 Jun 2019 03:26:56 +0000 (UTC) Received: by mo-csw.securemx.jp (mx-mo-csw1115) id x5P3QfIp021464; Tue, 25 Jun 2019 12:26:41 +0900 X-Iguazu-Qid: 2wGqtQ13rynb9UKsx1 X-Iguazu-QSIG: v=2; s=0; t=1561433201; q=2wGqtQ13rynb9UKsx1; m=ThdcnXO9KHnmXg6FtusPNiVcSnaXOFYqYlhtxqcuPk0= Received: from imx2.toshiba.co.jp (imx2.toshiba.co.jp [106.186.93.51]) by relay.securemx.jp (mx-mr1111) id x5P3QeOH001949; Tue, 25 Jun 2019 12:26:41 +0900 Received: from enc01.localdomain ([106.186.93.100]) by imx2.toshiba.co.jp with ESMTP id x5P3Qekq022266; Tue, 25 Jun 2019 12:26:40 +0900 (JST) Received: from hop001.toshiba.co.jp ([133.199.164.63]) by enc01.localdomain with ESMTP id x5P3QeC9026428; Tue, 25 Jun 2019 12:26:40 +0900 From: Daniel Sangorrin To: ben.hutchings@codethink.co.uk Date: Tue, 25 Jun 2019 12:26:36 +0900 X-TSB-HOP: ON Message-Id: <20190625032636.10694-7-daniel.sangorrin@toshiba.co.jp> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190625032636.10694-1-daniel.sangorrin@toshiba.co.jp> References: <20190625032636.10694-1-daniel.sangorrin@toshiba.co.jp> Cc: cip-dev@lists.cip-project.org Subject: [cip-dev] [cip-kernel-sec 6/6] pep8: fix pep8-related errors such as too long lines X-BeenThere: cip-dev@lists.cip-project.org X-Mailman-Version: 2.1.12 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Sender: cip-dev-bounces@lists.cip-project.org Errors-To: cip-dev-bounces@lists.cip-project.org X-Virus-Scanned: ClamAV using ClamSMTP These were distracting when checking new code. Signed-off-by: Daniel Sangorrin --- scripts/import_stable.py | 8 +++++--- scripts/kernel_sec/branch.py | 4 ++-- scripts/prepare_remotes.py | 2 +- scripts/report_affected.py | 2 +- scripts/webview.py | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/import_stable.py b/scripts/import_stable.py index 194219d..b4d8e0a 100755 --- a/scripts/import_stable.py +++ b/scripts/import_stable.py @@ -64,7 +64,8 @@ def get_backports(git_repo, remotes, branches, debug=False): backports.setdefault(mainline_commit, {})[branch_name] \ = stable_commit if line.strip() != '': - commit_re = BACKPORT_COMMIT_BOTTOM_RE # next line is not top + # next line is not top + commit_re = BACKPORT_COMMIT_BOTTOM_RE return backports @@ -120,7 +121,8 @@ def add_backports(branches, c_b_map, issue_commits, all_backports, if debug_context: print('%s/%s: recording commits' % (debug_context, branch_name)) - issue_commits.setdefault(branch_name, []).extend(branch_commits) + issue_commits.setdefault( + branch_name, []).extend(branch_commits) changed = True else: if debug_context: @@ -169,7 +171,7 @@ if __name__ == '__main__': metavar='DIRECTORY') parser.add_argument('--remote-name', dest='remote_name', action='append', default=[], - help='git remote name mappings, e.g. stable:korg-stable', + help='git remote name mappings, e.g. stable:mystable', metavar='NAME:OTHER-NAME') parser.add_argument('--mainline-remote', dest='mainline_remote_name', diff --git a/scripts/kernel_sec/branch.py b/scripts/kernel_sec/branch.py index ef88b54..9a7bc3a 100644 --- a/scripts/kernel_sec/branch.py +++ b/scripts/kernel_sec/branch.py @@ -62,8 +62,8 @@ def _extract_live_stable_branches(doc): if match: version = match.group(1) eol = match.group(2) is not None - if branch_type not in ['mainline', 'stable', 'longterm', 'linux-next'] \ - or version is None: + if branch_type not in ['mainline', 'stable', 'longterm', + 'linux-next'] or version is None: raise ValueError('failed to parse releases row text %r' % row_text) # Filter out irrelevant branches diff --git a/scripts/prepare_remotes.py b/scripts/prepare_remotes.py index 59310fc..ab0db24 100755 --- a/scripts/prepare_remotes.py +++ b/scripts/prepare_remotes.py @@ -45,7 +45,7 @@ if __name__ == '__main__': metavar='DIRECTORY') parser.add_argument('--remote-name', dest='remote_name', action='append', default=[], - help='git remote name mappings, e.g. stable:korg-stable', + help='git remote name mappings, e.g. stable:mystable', metavar='NAME:OTHER-NAME') parser.add_argument('--mainline-remote', dest='mainline_remote_name', diff --git a/scripts/report_affected.py b/scripts/report_affected.py index 32e9345..0296649 100755 --- a/scripts/report_affected.py +++ b/scripts/report_affected.py @@ -121,7 +121,7 @@ if __name__ == '__main__': metavar='DIRECTORY') parser.add_argument('--remote-name', dest='remote_name', action='append', default=[], - help='git remote name mappings, e.g. stable:korg-stable', + help='git remote name mappings, e.g. stable:mystable', metavar='NAME:OTHER-NAME') parser.add_argument('--mainline-remote', dest='mainline_remote_name', diff --git a/scripts/webview.py b/scripts/webview.py index 52a7b15..cf54948 100755 --- a/scripts/webview.py +++ b/scripts/webview.py @@ -205,7 +205,7 @@ if __name__ == '__main__': metavar='DIRECTORY') parser.add_argument('--remote-name', dest='remote_name', action='append', default=[], - help='git remote name mappings, e.g. stable:korg-stable', + help='git remote name mappings, e.g. stable:mystable', metavar='NAME:OTHER-NAME') parser.add_argument('--mainline-remote', dest='mainline_remote_name',