From patchwork Sun Oct 17 18:57:32 2010 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lucas Meneghel Rodrigues X-Patchwork-Id: 260481 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter1.kernel.org (8.14.4/8.14.3) with ESMTP id o9HIve6B008161 for ; Sun, 17 Oct 2010 18:57:40 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S932540Ab0JQS5h (ORCPT ); Sun, 17 Oct 2010 14:57:37 -0400 Received: from mx1.redhat.com ([209.132.183.28]:36619 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S932531Ab0JQS5g (ORCPT ); Sun, 17 Oct 2010 14:57:36 -0400 Received: from int-mx01.intmail.prod.int.phx2.redhat.com (int-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.11]) by mx1.redhat.com (8.13.8/8.13.8) with ESMTP id o9HIvZeq003094 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=OK); Sun, 17 Oct 2010 14:57:35 -0400 Received: from autotest.virt.bos.redhat.com (autotest.virt.bos.redhat.com [10.16.72.47]) by int-mx01.intmail.prod.int.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id o9HIvYiF005642; Sun, 17 Oct 2010 14:57:34 -0400 From: Lucas Meneghel Rodrigues To: autotest@test.kernel.org Cc: kvm@vger.kernel.org, ehabkost@redhat.com, Lucas Meneghel Rodrigues Subject: KVM test: Refactor Koji/Brew package download code Date: Sun, 17 Oct 2010 14:57:32 -0400 Message-Id: <1287341852-787-1-git-send-email-lmr@redhat.com> X-Scanned-By: MIMEDefang 2.67 on 10.5.11.11 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.3 (demeter1.kernel.org [140.211.167.41]); Sun, 17 Oct 2010 18:57:40 +0000 (UTC) diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py index a2b0a3f..dedb032 100644 --- a/client/tests/kvm/kvm_utils.py +++ b/client/tests/kvm/kvm_utils.py @@ -5,10 +5,15 @@ KVM test utility functions. """ import time, string, random, socket, os, signal, re, logging, commands, cPickle -import fcntl, shelve -from autotest_lib.client.bin import utils +import fcntl, shelve, ConfigParser +from autotest_lib.client.bin import utils, os_dep from autotest_lib.client.common_lib import error, logging_config import kvm_subprocess +try: + import koji + KOJI_INSTALLED = True +except ImportError: + KOJI_INSTALLED = False def dump_env(obj, filename): @@ -1401,3 +1406,141 @@ class PciAssignable(object): logging.info("Released device %s successfully", pci_id) except: return + + +class KojiDownloader(object): + """ + Stablish a connection with the build system, either koji or brew. + + This class provides a convenience methods to retrieve packages hosted on + the build system. + """ + def __init__(self, cmd): + """ + Verifies whether the system has koji or brew installed, then loads + the configuration file that will be used to download the files. + + @param cmd: Command name, either 'brew' or 'koji'. It is important + to figure out the appropriate configuration used by the + downloader. + @param dst_dir: Destination dir for the packages. + """ + if not KOJI_INSTALLED: + raise ValueError('No koji/brew installed on the machine') + + if os.path.isfile(cmd): + koji_cmd = cmd + else: + koji_cmd = os_dep.command(cmd) + + logging.debug("Found %s as the buildsystem interface", koji_cmd) + + config_map = {'/usr/bin/koji': '/etc/koji.conf', + '/usr/bin/brew': '/etc/brewkoji.conf'} + + try: + config_file = config_map[koji_cmd] + except IndexError: + raise ValueError('Could not find config file for %s' % koji_cmd) + + base_name = os.path.basename(koji_cmd) + if os.access(config_file, os.F_OK): + f = open(config_file) + config = ConfigParser.ConfigParser() + config.readfp(f) + f.close() + else: + raise IOError('Configuration file %s missing or with wrong ' + 'permissions' % config_file) + + if config.has_section(base_name): + self.koji_options = {} + session_options = {} + server = None + for name, value in config.items(base_name): + if name in ('user', 'password', 'debug_xmlrpc', 'debug'): + session_options[name] = value + self.koji_options[name] = value + self.session = koji.ClientSession(self.koji_options['server'], + session_options) + else: + raise ValueError('Koji config file %s does not have a %s ' + 'session' % (config_file, base_name)) + + + def get(self, src_package, dst_dir, rfilter=None, tag=None, build=None, + arch=None): + """ + Download a list of packages from the build system. + + This will download all packages originated from source package [package] + with given [tag] or [build] for the architecture reported by the + machine. + + @param src_package: Source package name. + @param dst_dir: Destination directory for the downloaded packages. + @param rfilter: Regexp filter, only download the packages that match + that particular filter. + @param tag: Build system tag. + @param build: Build system ID. + @param arch: Package arch. Useful when you want to download noarch + packages. + + @return: List of paths with the downloaded rpm packages. + """ + if build and build.isdigit(): + build = int(build) + + if tag and build: + logging.info("Both tag and build parameters provided, ignoring tag " + "parameter...") + + if not tag and not build: + raise ValueError("Koji install selected but neither koji_tag " + "nor koji_build parameters provided. Please " + "provide an appropriate tag or build name.") + + if not build: + builds = self.session.listTagged(tag, latest=True, + package=src_package) + if not builds: + raise ValueError("Tag %s has no builds of %s" % (tag, + src_package)) + info = builds[0] + else: + info = self.session.getBuild(build) + + if info is None: + raise ValueError('No such brew/koji build: %s' % build) + + if arch is None: + arch = utils.get_arch() + + rpms = self.session.listRPMs(buildID=info['id'], + arches=arch) + if not rpms: + raise ValueError("No %s packages available for %s" % + arch, koji.buildLabel(info)) + + rpm_paths = [] + for rpm in rpms: + rpm_name = koji.pathinfo.rpm(rpm) + url = ("%s/%s/%s/%s/%s" % (self.koji_options['pkgurl'], + info['package_name'], + info['version'], info['release'], + rpm_name)) + if rfilter: + filter_regexp = re.compile(rfilter, re.IGNORECASE) + if filter_regexp.match(os.path.basename(rpm_name)): + download = True + else: + download = False + else: + download = True + + if download: + r = utils.get_file(url, + os.path.join(dst_dir, os.path.basename(url))) + rpm_paths.append(r) + + return rpm_paths diff --git a/client/tests/kvm/tests/build.py b/client/tests/kvm/tests/build.py index 77fdc9e..d84c820 100644 --- a/client/tests/kvm/tests/build.py +++ b/client/tests/kvm/tests/build.py @@ -1,10 +1,5 @@ import time, os, sys, urllib, re, signal, logging, datetime, glob, ConfigParser import shutil -try: - import koji - KOJI_INSTALLED = True -except ImportError: - KOJI_INSTALLED = False from autotest_lib.client.bin import utils, test, os_dep from autotest_lib.client.common_lib import error import kvm_utils @@ -268,96 +263,27 @@ class KojiInstaller(YumInstaller): """ def __init__(self, test, params): """ - Initialize koji/brew session. + Gets parameters and initializes the package downloader. @param test: kvm test object @param params: Dictionary with test arguments """ super(KojiInstaller, self).__init__(test, params) - default_koji_cmd = '/usr/bin/koji' default_src_pkg = 'qemu' - - self.koji_cmd = params.get("koji_cmd", default_koji_cmd) self.src_pkg = params.get("src_pkg", default_src_pkg) - - # Checking if all required dependencies are available - os_dep.command(self.koji_cmd) - - config_map = {'/usr/bin/koji': '/etc/koji.conf', - '/usr/bin/brew': '/etc/brewkoji.conf'} - config_file = config_map[self.koji_cmd] - base_name = os.path.basename(self.koji_cmd) - if os.access(config_file, os.F_OK): - f = open(config_file) - config = ConfigParser.ConfigParser() - config.readfp(f) - f.close() - else: - raise error.TestError('Configuration file %s missing or with wrong ' - 'permissions' % config_file) - - if config.has_section(base_name): - self.koji_options = {} - session_options = {} - server = None - for name, value in config.items(base_name): - if name in ('user', 'password', 'debug_xmlrpc', 'debug'): - session_options[name] = value - self.koji_options[name] = value - self.session = koji.ClientSession(self.koji_options['server'], - session_options) - else: - raise error.TestError('Koji config file %s does not have a %s ' - 'session' % (config_file, base_name)) - self.tag = params.get("koji_tag", None) self.build = params.get("koji_build", None) - if self.build and self.build.isdigit(): - self.build = int(self.build) - if self.tag and self.build: - logging.info("Both tag and build parameters provided, ignoring tag " - "parameter...") - if not self.tag and not self.build: - raise error.TestError("Koji install selected but neither koji_tag " - "nor koji_build parameters provided. Please " - "provide an appropriate tag or build name.") + koji_cmd = params.get("koji_cmd", default_koji_cmd) + self.downloader = kvm_utils.KojiDownloader(cmd=koji_cmd) def _get_packages(self): """ Downloads the specific arch RPMs for the specific build name. """ - if self.build is None: - try: - builds = self.session.listTagged(self.tag, latest=True, - package=self.src_pkg) - except koji.GenericError, e: - raise error.TestError("Error finding latest build for tag %s: " - "%s" % (self.tag, e)) - if not builds: - raise error.TestError("Tag %s has no builds of %s" % - (self.tag, self.src_pkg)) - info = builds[0] - else: - info = self.session.getBuild(self.build) - - if info is None: - raise error.TestError('No such brew/koji build: %s' % - self.build) - rpms = self.session.listRPMs(buildID=info['id'], - arches=utils.get_arch()) - if not rpms: - raise error.TestError("No %s packages available for %s" % - utils.get_arch(), koji.buildLabel(info)) - for rpm in rpms: - rpm_name = koji.pathinfo.rpm(rpm) - url = ("%s/%s/%s/%s/%s" % (self.koji_options['pkgurl'], - info['package_name'], - info['version'], info['release'], - rpm_name)) - utils.get_file(url, - os.path.join(self.srcdir, os.path.basename(url))) + self.downloader.get(src_package=self.src_pkg, tag=self.tag, + build=self.build, dst_dir=self.srcdir) def install(self): @@ -695,11 +621,7 @@ def run_build(test, params, env): elif install_mode == 'yum': installer = YumInstaller(test, params) elif install_mode == 'koji': - if KOJI_INSTALLED: - installer = KojiInstaller(test, params) - else: - raise error.TestError('Koji install selected but koji/brew are not ' - 'installed') + installer = KojiInstaller(test, params) else: raise error.TestError('Invalid or unsupported' ' install mode: %s' % install_mode)