From patchwork Mon Oct 5 17:40:31 2009 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lucas Meneghel Rodrigues X-Patchwork-Id: 51761 Received: from vger.kernel.org (vger.kernel.org [209.132.176.167]) by demeter.kernel.org (8.14.2/8.14.2) with ESMTP id n95Hj5ga006407 for ; Mon, 5 Oct 2009 17:45:05 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752306AbZJERlH (ORCPT ); Mon, 5 Oct 2009 13:41:07 -0400 Received: (majordomo@vger.kernel.org) by vger.kernel.org id S1752406AbZJERlH (ORCPT ); Mon, 5 Oct 2009 13:41:07 -0400 Received: from mx1.redhat.com ([209.132.183.28]:54041 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752306AbZJERlF (ORCPT ); Mon, 5 Oct 2009 13:41:05 -0400 Received: from int-mx05.intmail.prod.int.phx2.redhat.com (int-mx05.intmail.prod.int.phx2.redhat.com [10.5.11.18]) by mx1.redhat.com (8.13.8/8.13.8) with ESMTP id n95HeYql013583; Mon, 5 Oct 2009 13:40:35 -0400 Received: from localhost.localdomain (vpn-12-223.rdu.redhat.com [10.11.12.223]) by int-mx05.intmail.prod.int.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id n95HeVpc001361; Mon, 5 Oct 2009 13:40:32 -0400 From: Lucas Meneghel Rodrigues To: autotest@test.kernel.org Cc: kvm@vger.kernel.org, jadmanski@google.com, Lucas Meneghel Rodrigues Subject: [PATCH] Refactor of setup_modules.py Date: Mon, 5 Oct 2009 14:40:31 -0300 Message-Id: <1254764431-11210-1-git-send-email-lmr@redhat.com> X-Scanned-By: MIMEDefang 2.67 on 10.5.11.18 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org diff --git a/client/setup_modules.py b/client/setup_modules.py index dc255c4..188114c 100644 --- a/client/setup_modules.py +++ b/client/setup_modules.py @@ -1,4 +1,9 @@ -__author__ = "jadmanski@google.com (John Admanski)" +""" +Utility library used for autotest library namespace configuration. + +@copyright Google 2007-2009 +@author John Admanski +""" import os, sys @@ -12,134 +17,195 @@ check_version.check_python_version() import new, glob, traceback +class namespace_config(object): + """ + Class to perform namespace configuration for autotest. -def _create_module(name): - """Create a single top-level module""" - module = new.module(name) - sys.modules[name] = module - return module - - -def _create_module_and_parents(name): - """Create a module, and all the necessary parents""" - parts = name.split(".") - # first create the top-level module - parent = _create_module(parts[0]) - created_parts = [parts[0]] - parts.pop(0) - # now, create any remaining child modules - while parts: - child_name = parts.pop(0) - module = new.module(child_name) - setattr(parent, child_name, module) - created_parts.append(child_name) - sys.modules[".".join(created_parts)] = module - parent = module - - -def _import_children_into_module(parent_module_name, path): - """Import all the packages on a path into a parent module""" - # find all the packages at 'path' - names = [] - for filename in os.listdir(path): - full_name = os.path.join(path, filename) - if not os.path.isdir(full_name): - continue # skip files - if "." in filename: - continue # if "." is in the name it's not a valid package name - if not os.access(full_name, os.R_OK | os.X_OK): - continue # need read + exec access to make a dir importable - if "__init__.py" in os.listdir(full_name): - names.append(filename) - # import all the packages and insert them into 'parent_module' - sys.path.insert(0, path) - for name in names: - module = __import__(name) - # add the package to the parent - parent_module = sys.modules[parent_module_name] - setattr(parent_module, name, module) - full_name = parent_module_name + "." + name - sys.modules[full_name] = module - # restore the system path - sys.path.pop(0) + This is what makes possible to use statements like + from autotest_lib.client.bin import package + """ + def __init__(self, base_path, root_module_name=""): + """ + Perform all the necessary setup so that all the packages at + 'base_path' can be imported via "import root_module_name.package". + If root_module_name is empty, then all the packages at base_path + are inserted as top-level packages. + + Also, setup all the common.* aliases for modules in the common + library. + + The setup must be different if you are running on an Autotest server + or on a test machine that just has the client directories installed. + + @param base_path: Path used to compute all relative paths needed for + the setup. + @param root_module_name: Top level name of the 'virtual' library + namespace. + """ + # Hack... Any better ideas? + if (root_module_name == 'autotest_lib.client' and + os.path.exists(os.path.join(os.path.dirname(__file__), + '..', 'server'))): + root_module_name = 'autotest_lib' + base_path = os.path.abspath(os.path.join(base_path, '..')) + self.root_module_name = root_module_name + self.base_path = base_path + + + def _create_module(self, name): + """ + Create a single top-level module. + + @param name: Name of the module you wish to create. + """ + module = new.module(name) + sys.modules[name] = module + return module + + + def _create_module_and_parents(self, name): + """ + Create a module, and all the necessary parents. + + @param name: Name of the module you wish to create. + """ + parts = name.split(".") + # first create the top-level module + parent = self._create_module(parts[0]) + created_parts = [parts[0]] + parts.pop(0) + # now, create any remaining child modules + while parts: + child_name = parts.pop(0) + module = new.module(child_name) + setattr(parent, child_name, module) + created_parts.append(child_name) + sys.modules[".".join(created_parts)] = module + parent = module + + + def _import_children_into_module(self, parent_module_name, path): + """ + Import all the packages on a path into a parent module. + + @param parent_module_name: Name of the parent module you want to use + @param path: Path that contains a number of python packages (libs) + """ + # find all the packages at 'path' + names = [] + for filename in os.listdir(path): + full_name = os.path.join(path, filename) + if not os.path.isdir(full_name): + continue # skip files + if "." in filename: + continue # if "." is in the name it's not a valid package name + if not os.access(full_name, os.R_OK | os.X_OK): + continue # need read + exec access to make a dir importable + if "__init__.py" in os.listdir(full_name): + names.append(filename) + # import all the packages and insert them into 'parent_module' + sys.path.insert(0, path) + for name in names: + module = __import__(name) + # add the package to the parent + parent_module = sys.modules[parent_module_name] + setattr(parent_module, name, module) + full_name = parent_module_name + "." + name + sys.modules[full_name] = module + # restore the system path + sys.path.pop(0) + + + def _autotest_logging_handle_error(self, record): + """ + Method to monkey patch into logging.Handler to replace handleError. + + @param record: Logging record that is going to be treated. + """ + # The same as the default logging.Handler.handleError but also prints + # out the original record causing the error so there is -some- idea + # about which call caused the logging error. + import logging + if logging.raiseExceptions: + # Avoid recursion as the below output can end up back in here when + # something has *seriously* gone wrong in autotest. + logging.raiseExceptions = 0 + sys.stderr.write('Exception occurred formatting message: ' + '%r using args %r\n' % (record.msg, record.args)) + traceback.print_stack() + sys.stderr.write('Future logging formatting exceptions disabled.\n') + + if self.root_module_name == 'autotest_lib': + # Allow locally installed third party packages to be found + # before any that are installed on the system itself when not. + # running as a client. + # This is primarily for the benefit of frontend and tko so that they + # may use libraries other than those available as system packages. + sys.path.insert(0, os.path.join(self.base_path, "site-packages")) + + + def _monkeypatch_logging_handle_error(self): + """ + Hack out logging.py to introduce a custom logging error handling + function. + """ + # Hack out logging.py* + logging_py = os.path.join(os.path.dirname(__file__), "common_lib", + "logging.py*") + if glob.glob(logging_py): + os.system("rm -f %s" % logging_py) + + # Monkey patch our own handleError into the logging module's + # StreamHandler. A nicer way of doing this -might- be to have our own + # logging module define an autotest Logger instance that added our own + # Handler subclass with this handleError method in it. But that would + # mean modifying tons of code. + import logging + assert callable(logging.Handler.handleError) + logging.Handler.handleError = self._autotest_logging_handle_error + + + def setup(self): + """ + Effectively creates the autotest_lib namespace. + """ + # Some internal functions might need to reference those variables. + # It doesn't occur to me how to refactor the API in this file in a clean + # way to avoid + self._create_module_and_parents(self.root_module_name) + self._import_children_into_module(self.root_module_name, self.base_path) + + if self.root_module_name == 'autotest_lib': + # Allow locally installed third party packages to be found + # before any that are installed on the system itself when not. + # running as a client. + # This is primarily for the benefit of frontend and tko so that they + # may use libraries other than those available as system packages. + sys.path.insert(0, os.path.join(self.base_path, "site-packages")) + + self._monkeypatch_logging_handle_error() def import_module(module, from_where): - """Equivalent to 'from from_where import module' - Returns the corresponding module""" + """ + Equivalent to 'from from_where import module' + + @param module: Module intended to be loaded + @param from_where: Top level module where we want to import the module + @return: loaded module + """ from_module = __import__(from_where, globals(), locals(), [module]) return getattr(from_module, module) -def _autotest_logging_handle_error(self, record): - """Method to monkey patch into logging.Handler to replace handleError.""" - # The same as the default logging.Handler.handleError but also prints - # out the original record causing the error so there is -some- idea - # about which call caused the logging error. - import logging - if logging.raiseExceptions: - # Avoid recursion as the below output can end up back in here when - # something has *seriously* gone wrong in autotest. - logging.raiseExceptions = 0 - sys.stderr.write('Exception occurred formatting message: ' - '%r using args %r\n' % (record.msg, record.args)) - traceback.print_stack() - sys.stderr.write('Future logging formatting exceptions disabled.\n') - - if root_module_name == 'autotest_lib': - # Allow locally installed third party packages to be found - # before any that are installed on the system itself when not. - # running as a client. - # This is primarily for the benefit of frontend and tko so that they - # may use libraries other than those available as system packages. - sys.path.insert(0, os.path.join(base_path, "site-packages")) - - -def _monkeypatch_logging_handle_error(): - # Hack out logging.py* - logging_py = os.path.join(os.path.dirname(__file__), "common_lib", - "logging.py*") - if glob.glob(logging_py): - os.system("rm -f %s" % logging_py) - - # Monkey patch our own handleError into the logging module's StreamHandler. - # A nicer way of doing this -might- be to have our own logging module define - # an autotest Logger instance that added our own Handler subclass with this - # handleError method in it. But that would mean modifying tons of code. - import logging - assert callable(logging.Handler.handleError) - logging.Handler.handleError = _autotest_logging_handle_error - - def setup(base_path, root_module_name=""): """ - Perform all the necessary setup so that all the packages at - 'base_path' can be imported via "import root_module_name.package". - If root_module_name is empty, then all the packages at base_path - are inserted as top-level packages. - - Also, setup all the common.* aliases for modules in the common - library. + Configures the autotest_lib namespace for use on a given autotest + entry point - The setup must be different if you are running on an Autotest server - or on a test machine that just has the client directories installed. + @param base_path: Path used to compute all relative paths needed for the + setup. + @param root_module_name: Top level name of the 'virtual' library namespace. """ - # Hack... Any better ideas? - if (root_module_name == 'autotest_lib.client' and - os.path.exists(os.path.join(os.path.dirname(__file__), - '..', 'server'))): - root_module_name = 'autotest_lib' - base_path = os.path.abspath(os.path.join(base_path, '..')) - - _create_module_and_parents(root_module_name) - _import_children_into_module(root_module_name, base_path) - - if root_module_name == 'autotest_lib': - # Allow locally installed third party packages to be found - # before any that are installed on the system itself when not. - # running as a client. - # This is primarily for the benefit of frontend and tko so that they - # may use libraries other than those available as system packages. - sys.path.insert(0, os.path.join(base_path, "site-packages")) - - _monkeypatch_logging_handle_error() + configurator = namespace_config(base_path, root_module_name) + configurator.setup()