From patchwork Wed Dec 29 22:37:26 2010 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lucas Meneghel Rodrigues X-Patchwork-Id: 440071 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter1.kernel.org (8.14.4/8.14.3) with ESMTP id oBUMBElS000490 for ; Thu, 30 Dec 2010 22:11:59 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754081Ab0L2Who (ORCPT ); Wed, 29 Dec 2010 17:37:44 -0500 Received: from mx1.redhat.com ([209.132.183.28]:17625 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1753535Ab0L2Whn (ORCPT ); Wed, 29 Dec 2010 17:37:43 -0500 Received: from int-mx10.intmail.prod.int.phx2.redhat.com (int-mx10.intmail.prod.int.phx2.redhat.com [10.5.11.23]) by mx1.redhat.com (8.13.8/8.13.8) with ESMTP id oBTMbfNq007062 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=OK); Wed, 29 Dec 2010 17:37:41 -0500 Received: from freedom.redhat.com (vpn-8-145.rdu.redhat.com [10.11.8.145]) by int-mx10.intmail.prod.int.phx2.redhat.com (8.14.4/8.14.4) with ESMTP id oBTMbWiu012882; Wed, 29 Dec 2010 17:37:40 -0500 From: Lucas Meneghel Rodrigues To: autotest@test.kernel.org Cc: kvm@vger.kernel.org, Lucas Meneghel Rodrigues Subject: [PATCH 4/8] KVM test: Make the build test to use the installer library Date: Wed, 29 Dec 2010 20:37:26 -0200 Message-Id: <1293662250-18292-5-git-send-email-lmr@redhat.com> In-Reply-To: <1293662250-18292-1-git-send-email-lmr@redhat.com> References: <1293662250-18292-1-git-send-email-lmr@redhat.com> X-Scanned-By: MIMEDefang 2.68 on 10.5.11.23 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.3 (demeter1.kernel.org [140.211.167.41]); Thu, 30 Dec 2010 22:11:59 +0000 (UTC) diff --git a/client/tests/kvm/tests/build.py b/client/tests/kvm/tests/build.py index bb3e2dc..1eef7a1 100644 --- a/client/tests/kvm/tests/build.py +++ b/client/tests/kvm/tests/build.py @@ -1,604 +1,4 @@ -import time, os, sys, urllib, re, signal, logging, datetime, glob, ConfigParser -import shutil -from autotest_lib.client.bin import utils, test, os_dep -from autotest_lib.client.common_lib import error -import kvm_utils - - -def check_configure_options(script_path): - """ - Return the list of available options (flags) of a given kvm configure build - script. - - @param script: Path to the configure script - """ - abspath = os.path.abspath(script_path) - help_raw = utils.system_output('%s --help' % abspath, ignore_status=True) - help_output = help_raw.split("\n") - option_list = [] - for line in help_output: - cleaned_line = line.lstrip() - if cleaned_line.startswith("--"): - option = cleaned_line.split()[0] - option = option.split("=")[0] - option_list.append(option) - - return option_list - - -def kill_qemu_processes(): - """ - Kills all qemu processes, also kills all processes holding /dev/kvm down. - """ - logging.debug("Killing any qemu processes that might be left behind") - utils.system("pkill qemu", ignore_status=True) - # Let's double check to see if some other process is holding /dev/kvm - if os.path.isfile("/dev/kvm"): - utils.system("fuser -k /dev/kvm", ignore_status=True) - - -def load_kvm_modules(module_dir=None, load_stock=False, extra_modules=None): - """ - Unload previously loaded kvm modules, then load modules present on any - sub directory of module_dir. Function will walk through module_dir until - it finds the modules. - - @param module_dir: Directory where the KVM modules are located. - @param load_stock: Whether we are going to load system kernel modules. - @param extra_modules: List of extra modules to load. - """ - vendor = "intel" - if os.system("grep vmx /proc/cpuinfo 1>/dev/null") != 0: - vendor = "amd" - logging.debug("Detected CPU vendor as '%s'" %(vendor)) - - kill_qemu_processes() - - logging.info("Unloading previously loaded KVM modules") - utils.unload_module("kvm") - if extra_modules: - for module in extra_modules: - utils.unload_module(module) - - if module_dir: - logging.info("Loading the built KVM modules...") - kvm_module_path = None - kvm_vendor_module_path = None - abort = False - - list_modules = ['kvm.ko', 'kvm-%s.ko' % vendor] - if extra_modules: - for extra_module in extra_modules: - list_modules.append('%s.ko' % extra_module) - - list_module_paths = [] - for folder, subdirs, files in os.walk(module_dir): - for module in list_modules: - if module in files: - module_path = os.path.join(folder, module) - list_module_paths.append(module_path) - - # We might need to arrange the modules in the correct order - # to avoid module load problems - list_modules_load = [] - for module in list_modules: - for module_path in list_module_paths: - if os.path.basename(module_path) == module: - list_modules_load.append(module_path) - - if len(list_module_paths) != len(list_modules): - logging.error("KVM modules not found. If you don't want to use the " - "modules built by this test, make sure the option " - "load_modules: 'no' is marked on the test control " - "file.") - raise error.TestError("The modules %s were requested to be loaded, " - "but the only modules found were %s" % - (list_modules, list_module_paths)) - - for module_path in list_modules_load: - try: - utils.system("insmod %s" % module_path) - except Exception, e: - raise error.TestFail("Failed to load KVM modules: %s" % e) - - if load_stock: - logging.info("Loading current system KVM modules...") - utils.system("modprobe kvm") - utils.system("modprobe kvm-%s" % vendor) - if extra_modules: - for module in extra_modules: - utils.system("modprobe %s" % module) - - -def create_symlinks(test_bindir, prefix=None, bin_list=None, unittest=None): - """ - Create symbolic links for the appropriate qemu and qemu-img commands on - the kvm test bindir. - - @param test_bindir: KVM test bindir - @param prefix: KVM prefix path - @param bin_list: List of qemu binaries to link - @param unittest: Path to configuration file unittests.cfg - """ - qemu_path = os.path.join(test_bindir, "qemu") - qemu_img_path = os.path.join(test_bindir, "qemu-img") - qemu_unittest_path = os.path.join(test_bindir, "unittests") - if os.path.lexists(qemu_path): - os.unlink(qemu_path) - if os.path.lexists(qemu_img_path): - os.unlink(qemu_img_path) - if unittest and os.path.lexists(qemu_unittest_path): - os.unlink(qemu_unittest_path) - - logging.debug("Linking qemu binaries") - - if bin_list: - for bin in bin_list: - if os.path.basename(bin) == 'qemu-kvm': - os.symlink(bin, qemu_path) - elif os.path.basename(bin) == 'qemu-img': - os.symlink(bin, qemu_img_path) - - elif prefix: - kvm_qemu = os.path.join(prefix, "bin", "qemu-system-x86_64") - if not os.path.isfile(kvm_qemu): - raise error.TestError('Invalid qemu path') - kvm_qemu_img = os.path.join(prefix, "bin", "qemu-img") - if not os.path.isfile(kvm_qemu_img): - raise error.TestError('Invalid qemu-img path') - os.symlink(kvm_qemu, qemu_path) - os.symlink(kvm_qemu_img, qemu_img_path) - - if unittest: - logging.debug("Linking unittest dir") - os.symlink(unittest, qemu_unittest_path) - - -def install_roms(rom_dir, prefix): - logging.debug("Path to roms specified. Copying roms to install prefix") - rom_dst_dir = os.path.join(prefix, 'share', 'qemu') - for rom_src in glob.glob('%s/*.bin' % rom_dir): - rom_dst = os.path.join(rom_dst_dir, os.path.basename(rom_src)) - logging.debug("Copying rom file %s to %s", rom_src, rom_dst) - shutil.copy(rom_src, rom_dst) - - -def save_build(build_dir, dest_dir): - logging.debug('Saving the result of the build on %s', dest_dir) - base_name = os.path.basename(build_dir) - tarball_name = base_name + '.tar.bz2' - os.chdir(os.path.dirname(build_dir)) - utils.system('tar -cjf %s %s' % (tarball_name, base_name)) - shutil.move(tarball_name, os.path.join(dest_dir, tarball_name)) - - -class BaseInstaller(object): - def __init__(self, test, params): - load_modules = params.get('load_modules', 'no') - if not load_modules or load_modules == 'yes': - self.load_modules = True - elif load_modules == 'no': - self.load_modules = False - default_extra_modules = str(None) - self.extra_modules = eval(params.get("extra_modules", - default_extra_modules)) - - self.srcdir = test.srcdir - if not os.path.isdir(self.srcdir): - os.makedirs(self.srcdir) - - self.test_bindir = test.bindir - self.results_dir = test.resultsdir - - # KVM build prefix, for the modes that do need it - prefix = os.path.join(test.bindir, 'build') - self.prefix = os.path.abspath(prefix) - - # Current host kernel directory - default_host_kernel_source = '/lib/modules/%s/build' % os.uname()[2] - self.host_kernel_srcdir = params.get('host_kernel_source', - default_host_kernel_source) - - # Extra parameters that can be passed to the configure script - self.extra_configure_options = params.get('extra_configure_options', - None) - - # Do we want to save the result of the build on test.resultsdir? - self.save_results = True - save_results = params.get('save_results', 'no') - if save_results == 'no': - self.save_results = False - - -class YumInstaller(BaseInstaller): - """ - Class that uses yum to install and remove packages. - """ - def __init__(self, test, params): - super(YumInstaller, self).__init__(test, params) - # Checking if all required dependencies are available - os_dep.command("rpm") - os_dep.command("yum") - - default_pkg_list = str(['qemu-kvm', 'qemu-kvm-tools']) - default_qemu_bin_paths = str(['/usr/bin/qemu-kvm', '/usr/bin/qemu-img']) - default_pkg_path_list = str(None) - self.pkg_list = eval(params.get("pkg_list", default_pkg_list)) - self.pkg_path_list = eval(params.get("pkg_path_list", - default_pkg_path_list)) - self.qemu_bin_paths = eval(params.get("qemu_bin_paths", - default_qemu_bin_paths)) - - - def _clean_previous_installs(self): - kill_qemu_processes() - removable_packages = "" - for pkg in self.pkg_list: - removable_packages += " %s" % pkg - - utils.system("yum remove -y %s" % removable_packages) - - - def _get_packages(self): - for pkg in self.pkg_path_list: - utils.get_file(pkg, os.path.join(self.srcdir, - os.path.basename(pkg))) - - - def _install_packages(self): - """ - Install all downloaded packages. - """ - os.chdir(self.srcdir) - utils.system("yum install --nogpgcheck -y *.rpm") - - - def install(self): - self._clean_previous_installs() - self._get_packages() - self._install_packages() - create_symlinks(test_bindir=self.test_bindir, - bin_list=self.qemu_bin_paths) - if self.load_modules: - load_kvm_modules(load_stock=True, extra_modules=self.extra_modules) - if self.save_results: - save_build(self.srcdir, self.results_dir) - - -class KojiInstaller(YumInstaller): - """ - Class that handles installing KVM from the fedora build service, koji. - It uses yum to install and remove packages. - """ - def __init__(self, test, params): - """ - Gets parameters and initializes the package downloader. - - @param test: kvm test object - @param params: Dictionary with test arguments - """ - super(KojiInstaller, self).__init__(test, params) - default_koji_cmd = '/usr/bin/koji' - default_src_pkg = 'qemu' - self.src_pkg = params.get("src_pkg", default_src_pkg) - self.tag = params.get("koji_tag", None) - self.build = params.get("koji_build", None) - koji_cmd = params.get("koji_cmd", default_koji_cmd) - self.downloader = kvm_utils.KojiDownloader(cmd=koji_cmd) - - - def _get_packages(self): - """ - Downloads the specific arch RPMs for the specific build name. - """ - self.downloader.get(src_package=self.src_pkg, tag=self.tag, - build=self.build, dst_dir=self.srcdir) - - - def install(self): - super(KojiInstaller, self)._clean_previous_installs() - self._get_packages() - super(KojiInstaller, self)._install_packages() - create_symlinks(test_bindir=self.test_bindir, - bin_list=self.qemu_bin_paths) - if self.load_modules: - load_kvm_modules(load_stock=True, extra_modules=self.extra_modules) - if self.save_results: - save_build(self.srcdir, self.results_dir) - - -class SourceDirInstaller(BaseInstaller): - """ - Class that handles building/installing KVM directly from a tarball or - a single source code dir. - """ - def __init__(self, test, params): - """ - Initializes class attributes, and retrieves KVM code. - - @param test: kvm test object - @param params: Dictionary with test arguments - """ - super(SourceDirInstaller, self).__init__(test, params) - - install_mode = params["mode"] - srcdir = params.get("srcdir", None) - self.path_to_roms = params.get("path_to_rom_images", None) - - if install_mode == 'localsrc': - if srcdir is None: - raise error.TestError("Install from source directory specified" - "but no source directory provided on the" - "control file.") - else: - shutil.copytree(srcdir, self.srcdir) - - if install_mode == 'release': - release_tag = params.get("release_tag") - release_dir = params.get("release_dir") - release_listing = params.get("release_listing") - logging.info("Installing KVM from release tarball") - if not release_tag: - release_tag = kvm_utils.get_latest_kvm_release_tag( - release_listing) - tarball = os.path.join(release_dir, 'kvm', release_tag, - "kvm-%s.tar.gz" % release_tag) - logging.info("Retrieving release kvm-%s" % release_tag) - tarball = utils.unmap_url("/", tarball, "/tmp") - - elif install_mode == 'snapshot': - logging.info("Installing KVM from snapshot") - snapshot_dir = params.get("snapshot_dir") - if not snapshot_dir: - raise error.TestError("Snapshot dir not provided") - snapshot_date = params.get("snapshot_date") - if not snapshot_date: - # Take yesterday's snapshot - d = (datetime.date.today() - - datetime.timedelta(1)).strftime("%Y%m%d") - else: - d = snapshot_date - tarball = os.path.join(snapshot_dir, "kvm-snapshot-%s.tar.gz" % d) - logging.info("Retrieving kvm-snapshot-%s" % d) - tarball = utils.unmap_url("/", tarball, "/tmp") - - elif install_mode == 'localtar': - tarball = params.get("tarball") - if not tarball: - raise error.TestError("KVM Tarball install specified but no" - " tarball provided on control file.") - logging.info("Installing KVM from a local tarball") - logging.info("Using tarball %s") - tarball = utils.unmap_url("/", params.get("tarball"), "/tmp") - - if install_mode in ['release', 'snapshot', 'localtar']: - utils.extract_tarball_to_dir(tarball, self.srcdir) - - if install_mode in ['release', 'snapshot', 'localtar', 'srcdir']: - self.repo_type = kvm_utils.check_kvm_source_dir(self.srcdir) - configure_script = os.path.join(self.srcdir, 'configure') - self.configure_options = check_configure_options(configure_script) - - - def _build(self): - make_jobs = utils.count_cpus() - os.chdir(self.srcdir) - # For testing purposes, it's better to build qemu binaries with - # debugging symbols, so we can extract more meaningful stack traces. - cfg = "./configure --prefix=%s" % self.prefix - if "--disable-strip" in self.configure_options: - cfg += " --disable-strip" - steps = [cfg, "make clean", "make -j %s" % make_jobs] - logging.info("Building KVM") - for step in steps: - utils.system(step) - - - def _install(self): - os.chdir(self.srcdir) - logging.info("Installing KVM userspace") - if self.repo_type == 1: - utils.system("make -C qemu install") - elif self.repo_type == 2: - utils.system("make install") - if self.path_to_roms: - install_roms(self.path_to_roms, self.prefix) - create_symlinks(self.test_bindir, self.prefix) - - - def _load_modules(self): - load_kvm_modules(module_dir=self.srcdir, - extra_modules=self.extra_modules) - - - def install(self): - self._build() - self._install() - if self.load_modules: - self._load_modules() - if self.save_results: - save_build(self.srcdir, self.results_dir) - - -class GitInstaller(SourceDirInstaller): - def __init__(self, test, params): - """ - Initialize class parameters and retrieves code from git repositories. - - @param test: kvm test object. - @param params: Dictionary with test parameters. - """ - super(GitInstaller, self).__init__(test, params) - - kernel_repo = params.get("git_repo") - user_repo = params.get("user_git_repo") - kmod_repo = params.get("kmod_repo") - test_repo = params.get("test_git_repo") - - kernel_branch = params.get("kernel_branch", "master") - user_branch = params.get("user_branch", "master") - kmod_branch = params.get("kmod_branch", "master") - test_branch = params.get("test_branch", "master") - - kernel_lbranch = params.get("kernel_lbranch", "master") - user_lbranch = params.get("user_lbranch", "master") - kmod_lbranch = params.get("kmod_lbranch", "master") - test_lbranch = params.get("test_lbranch", "master") - - kernel_commit = params.get("kernel_commit", None) - user_commit = params.get("user_commit", None) - kmod_commit = params.get("kmod_commit", None) - test_commit = params.get("test_commit", None) - - kernel_patches = eval(params.get("kernel_patches", "[]")) - user_patches = eval(params.get("user_patches", "[]")) - kmod_patches = eval(params.get("user_patches", "[]")) - - if not user_repo: - message = "KVM user git repository path not specified" - logging.error(message) - raise error.TestError(message) - - userspace_srcdir = os.path.join(self.srcdir, "kvm_userspace") - kvm_utils.get_git_branch(user_repo, user_branch, userspace_srcdir, - user_commit, user_lbranch) - self.userspace_srcdir = userspace_srcdir - - if user_patches: - os.chdir(self.userspace_srcdir) - for patch in user_patches: - utils.get_file(patch, os.path.join(self.userspace_srcdir, - os.path.basename(patch))) - utils.system('patch -p1 %s' % os.path.basename(patch)) - - if test_repo: - test_srcdir = os.path.join(self.srcdir, "kvm-unit-tests") - kvm_utils.get_git_branch(test_repo, test_branch, test_srcdir, - test_commit, test_lbranch) - unittest_cfg = os.path.join(test_srcdir, 'x86', - 'unittests.cfg') - self.test_srcdir = test_srcdir - else: - unittest_cfg = os.path.join(userspace_srcdir, 'kvm', 'test', 'x86', - 'unittests.cfg') - - self.unittest_cfg = None - if os.path.isfile(unittest_cfg): - self.unittest_cfg = unittest_cfg - - if kernel_repo: - kernel_srcdir = os.path.join(self.srcdir, "kvm") - kvm_utils.get_git_branch(kernel_repo, kernel_branch, kernel_srcdir, - kernel_commit, kernel_lbranch) - self.kernel_srcdir = kernel_srcdir - if kernel_patches: - os.chdir(self.kernel_srcdir) - for patch in kernel_patches: - utils.get_file(patch, os.path.join(self.userspace_srcdir, - os.path.basename(patch))) - utils.system('patch -p1 %s' % os.path.basename(patch)) - else: - self.kernel_srcdir = None - - if kmod_repo: - kmod_srcdir = os.path.join (self.srcdir, "kvm_kmod") - kvm_utils.get_git_branch(kmod_repo, kmod_branch, kmod_srcdir, - kmod_commit, kmod_lbranch) - self.kmod_srcdir = kmod_srcdir - if kmod_patches: - os.chdir(self.kmod_srcdir) - for patch in kmod_patches: - utils.get_file(patch, os.path.join(self.userspace_srcdir, - os.path.basename(patch))) - utils.system('patch -p1 %s' % os.path.basename(patch)) - else: - self.kmod_srcdir = None - - configure_script = os.path.join(self.userspace_srcdir, 'configure') - self.configure_options = check_configure_options(configure_script) - - - def _build(self): - make_jobs = utils.count_cpus() - cfg = './configure' - self.modules_build_succeed = False - if self.kmod_srcdir: - logging.info('Building KVM modules') - os.chdir(self.kmod_srcdir) - module_build_steps = [cfg, - 'make clean', - 'make sync LINUX=%s' % self.kernel_srcdir, - 'make'] - elif self.kernel_srcdir: - logging.info('Building KVM modules') - os.chdir(self.userspace_srcdir) - cfg += ' --kerneldir=%s' % self.host_kernel_srcdir - module_build_steps = [cfg, - 'make clean', - 'make -C kernel LINUX=%s sync' % self.kernel_srcdir] - else: - module_build_steps = [] - - try: - if module_build_steps: - for step in module_build_steps: - utils.run(step) - self.modules_build_succeed = True - except error.CmdError, e: - logging.error("KVM modules build failed to build: %s" % e) - - logging.info('Building KVM userspace code') - os.chdir(self.userspace_srcdir) - cfg += ' --prefix=%s' % self.prefix - if "--disable-strip" in self.configure_options: - cfg += ' --disable-strip' - if self.extra_configure_options: - cfg += ' %s' % self.extra_configure_options - utils.system(cfg) - utils.system('make clean') - utils.system('make -j %s' % make_jobs) - - self.unittest_prefix = None - if self.unittest_cfg: - os.chdir(os.path.dirname(os.path.dirname(self.unittest_cfg))) - utils.system('./configure --prefix=%s' % self.prefix) - utils.system('make') - utils.system('make install') - self.unittest_prefix = os.path.join(self.prefix, 'share', 'qemu', - 'tests') - - - def _install(self): - os.chdir(self.userspace_srcdir) - utils.system('make install') - if self.path_to_roms: - install_roms(self.path_to_roms, self.prefix) - create_symlinks(test_bindir=self.test_bindir, prefix=self.prefix, - bin_list=None, - unittest=self.unittest_prefix) - - - def _load_modules(self): - if self.kmod_srcdir and self.modules_build_succeed: - load_kvm_modules(module_dir=self.kmod_srcdir, - extra_modules=self.extra_modules) - elif self.kernel_srcdir and self.modules_build_succeed: - load_kvm_modules(module_dir=self.userspace_srcdir, - extra_modules=self.extra_modules) - else: - logging.info("Loading stock KVM modules") - load_kvm_modules(load_stock=True, - extra_modules=self.extra_modules) - - - def install(self): - self._build() - self._install() - if self.load_modules: - self._load_modules() - if self.save_results: - save_build(self.srcdir, self.results_dir) - +import installer def run_build(test, params, env): """ @@ -609,20 +9,17 @@ def run_build(test, params, env): @param params: Dictionary with test parameters. @param env: Test environment. """ - install_mode = params.get("mode") srcdir = params.get("srcdir", test.srcdir) params["srcdir"] = srcdir - if install_mode in ['localsrc', 'localtar', 'release', 'snapshot']: - installer = SourceDirInstaller(test, params) - elif install_mode == 'git': - installer = GitInstaller(test, params) - elif install_mode == 'yum': - installer = YumInstaller(test, params) - elif install_mode == 'koji': - installer = KojiInstaller(test, params) - else: - raise error.TestError('Invalid or unsupported' - ' install mode: %s' % install_mode) - - installer.install() + try: + installer_object = installer.make_installer(params) + installer_object.set_install_params(test, params) + installer_object.install() + env.register_installer(installer_object) + except Exception,e: + # if the build/install fails, don't allow other tests + # to get a installer. + msg = "KVM install failed: %s" % (e) + env.register_installer(installer.FailedInstaller(msg)) + raise