From patchwork Fri Sep 8 11:11:24 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Johannes Berg X-Patchwork-Id: 9943737 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork.web.codeaurora.org (Postfix) with ESMTP id 6B2946034B for ; Fri, 8 Sep 2017 11:11:30 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 5C9BD26861 for ; Fri, 8 Sep 2017 11:11:30 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 516162843C; Fri, 8 Sep 2017 11:11:30 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-6.9 required=2.0 tests=BAYES_00,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 2FC9226861 for ; Fri, 8 Sep 2017 11:11:29 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751675AbdIHLL2 (ORCPT ); Fri, 8 Sep 2017 07:11:28 -0400 Received: from s3.sipsolutions.net ([5.9.151.49]:33926 "EHLO sipsolutions.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751577AbdIHLL2 (ORCPT ); Fri, 8 Sep 2017 07:11:28 -0400 Received: by sipsolutions.net with esmtpsa (TLS1.2:ECDHE_RSA_AES_128_GCM_SHA256:128) (Exim 4.89) (envelope-from ) id 1dqHC1-0002c0-TV; Fri, 08 Sep 2017 13:11:25 +0200 From: Johannes Berg To: backports@vger.kernel.org Cc: Johannes Berg Subject: [PATCH] use spatch's built-in concurrency Date: Fri, 8 Sep 2017 13:11:24 +0200 Message-Id: <20170908111124.26455-1-johannes@sipsolutions.net> X-Mailer: git-send-email 2.14.1 Sender: backports-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: backports@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP From: Johannes Berg Signed-off-by: Johannes Berg --- devel/pycocci | 174 ---------------------------------------------------------- gentree.py | 16 ++++-- 2 files changed, 11 insertions(+), 179 deletions(-) delete mode 100755 devel/pycocci diff --git a/devel/pycocci b/devel/pycocci deleted file mode 100755 index 4aa1d3638347..000000000000 --- a/devel/pycocci +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2014 Luis R. Rodriguez -# Copyright (c) 2013 Johannes Berg -# -# This file is released under the GPLv2. -# -# Python wrapper for Coccinelle for multithreaded support, -# designed to be used for working on a git tree, and with sensible -# defaults, specifically for kernel developers. - -from multiprocessing import Process, cpu_count, Queue -import argparse, subprocess, os, sys -import tempfile, shutil - -# simple tempdir wrapper object for 'with' statement -# -# Usage: -# with tempdir.tempdir() as tmpdir: -# os.chdir(tmpdir) -# do something -# -class tempdir(object): - def __init__(self, suffix='', prefix='', dir=None, nodelete=False): - self.suffix = '' - self.prefix = '' - self.dir = dir - self.nodelete = nodelete - - def __enter__(self): - self._name = tempfile.mkdtemp(suffix=self.suffix, - prefix=self.prefix, - dir=self.dir) - return self._name - - def __exit__(self, type, value, traceback): - if self.nodelete: - print('not deleting directory %s!' % self._name) - else: - shutil.rmtree(self._name) - -class CoccinelleError(Exception): - pass - -class ExecutionErrorThread(CoccinelleError): - def __init__(self, errcode, fn, cocci_file, threads, t, logwrite, print_name): - self.error_code = errcode - logwrite("Failed to apply changes from %s\n" % print_name) - - logwrite("Specific log output from change that failed using %s\n" % print_name) - tf = open(fn, 'r') - for line in tf.read(): - logwrite(line) - tf.close() - - logwrite("Full log using %s\n" % print_name) - for num in range(threads): - fn = os.path.join(t, '.tmp_spatch_worker.' + str(num)) - if (not os.path.isfile(fn)): - continue - tf = open(fn, 'r') - for line in tf.read(): - logwrite(line) - tf.close() - os.unlink(fn) - -def spatch(cocci_file, outdir, - max_threads, thread_id, temp_dir, ret_q, extra_args=[]): - cmd = ['spatch', - '--sp-file', cocci_file, - '--in-place', - '--recursive-includes', - '--relax-include-path', - '--use-coccigrep', - '--timeout', '120', - '--dir', outdir ] - - if (max_threads > 1): - cmd.extend(['-max', str(max_threads), '-index', str(thread_id)]) - - cmd.extend(extra_args) - - fn = os.path.join(temp_dir, '.tmp_spatch_worker.' + str(thread_id)) - outfile = open(fn, 'w') - logwrite("%s\n" % " ".join(cmd)) - - sprocess = subprocess.Popen(cmd, - stdout=outfile, stderr=subprocess.STDOUT, - close_fds=True, universal_newlines=True) - sprocess.wait() - outfile.close() - ret_q.put((sprocess.returncode, fn)) - -def threaded_spatch(cocci_file, outdir, logwrite, num_jobs, - print_name, extra_args=[]): - num_cpus = cpu_count() - if num_jobs: - threads = int(num_jobs) - else: - threads = num_cpus - jobs = list() - output = "" - ret_q = Queue() - with tempdir() as t: - for num in range(threads): - p = Process(target=spatch, args=(cocci_file, outdir, - threads, num, t, ret_q, - extra_args)) - jobs.append(p) - for p in jobs: - p.start() - - for num in range(threads): - ret, fn = ret_q.get() - if ret != 0: - raise ExecutionErrorThread(ret, fn, cocci_file, threads, t, - logwrite, print_name) - for job in jobs: - p.join() - - for num in range(threads): - fn = os.path.join(t, '.tmp_spatch_worker.' + str(num)) - tf = open(fn, 'r') - output = output + tf.read() - tf.close() - os.unlink(fn) - return output - -def logwrite(msg): - sys.stdout.write(msg) - sys.stdout.flush() - -def _main(): - parser = argparse.ArgumentParser(description='Multithreaded Python wrapper for Coccinelle ' + - 'with sensible defaults, targetted specifically ' + - 'for git development environments') - parser.add_argument('cocci_file', metavar='', type=str, - help='This is the Coccinelle file you want to use') - parser.add_argument('target_dir', metavar='', type=str, - help='Target source directory to modify') - parser.add_argument('-p', '--profile-cocci', const=True, default=False, action="store_const", - help='Enable profile, this will pass --profile to Coccinelle.') - parser.add_argument('-j', '--jobs', metavar='', type=str, default=None, - help='Only use the cocci file passed for Coccinelle, don\'t do anything else, ' + - 'also creates a git repo on the target directory for easy inspection ' + - 'of changes done by Coccinelle.') - parser.add_argument('-v', '--verbose', const=True, default=False, action="store_const", - help='Enable output from Coccinelle') - args = parser.parse_args() - - if not os.path.isfile(args.cocci_file): - return -2 - - extra_spatch_args = [] - if args.profile_cocci: - extra_spatch_args.append('--profile') - jobs = 0 - if args.jobs > 0: - jobs = args.jobs - - output = threaded_spatch(args.cocci_file, - args.target_dir, - logwrite, - jobs, - os.path.basename(args.cocci_file), - extra_args=extra_spatch_args) - if args.verbose: - logwrite(output) - return 0 - -if __name__ == '__main__': - ret = _main() - if ret: - sys.exit(ret) diff --git a/gentree.py b/gentree.py index 026afdc2e02d..7d3f4a4f7b8f 100755 --- a/gentree.py +++ b/gentree.py @@ -5,6 +5,7 @@ import argparse, sys, os, errno, shutil, re, subprocess import tarfile, gzip +from multiprocessing import cpu_count # find self source_dir = os.path.abspath(os.path.dirname(__file__)) @@ -584,13 +585,18 @@ def apply_patches(args, desc, source_dir, patch_src, target_dir, logwrite=lambda prefix_len = len(os.path.join(source_dir, patch_src)) + 1 for cocci_file in sempatches: - # Until Coccinelle picks this up - pycocci = os.path.join(source_dir, 'devel/pycocci') - cmd = [pycocci, cocci_file] + cmd = ['spatch', + '--sp-file', cocci_file, + '--in-place', + '--recursive-includes', + '--relax-include-path', + '--use-coccigrep', + '--timeout', '120', + '-j', '%d' % cpu_count(), + '--dir', os.path.abspath(target_dir) ] extra_spatch_args = [] if args.profile_cocci: - cmd.append('--profile-cocci') - cmd.append(os.path.abspath(target_dir)) + cmd.append('--profile') print_name = cocci_file[prefix_len:] if args.verbose: logwrite("Applying SmPL patch %s" % print_name)