From patchwork Mon Nov 29 02:44:52 2010 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lucas Meneghel Rodrigues X-Patchwork-Id: 362772 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter1.kernel.org (8.14.4/8.14.3) with ESMTP id oAT2jFDT015119 for ; Mon, 29 Nov 2010 02:45:16 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754529Ab0K2CpI (ORCPT ); Sun, 28 Nov 2010 21:45:08 -0500 Received: from mx1.redhat.com ([209.132.183.28]:59046 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1754112Ab0K2CpH (ORCPT ); Sun, 28 Nov 2010 21:45:07 -0500 Received: from int-mx10.intmail.prod.int.phx2.redhat.com (int-mx10.intmail.prod.int.phx2.redhat.com [10.5.11.23]) by mx1.redhat.com (8.13.8/8.13.8) with ESMTP id oAT2j1RJ022736 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=OK); Sun, 28 Nov 2010 21:45:01 -0500 Received: from freedom.redhat.com (vpn-10-204.rdu.redhat.com [10.11.10.204]) by int-mx10.intmail.prod.int.phx2.redhat.com (8.14.4/8.14.4) with ESMTP id oAT2it22032739; Sun, 28 Nov 2010 21:44:57 -0500 From: Lucas Meneghel Rodrigues To: autotest@test.kernel.org Cc: kvm@vger.kernel.org, Lucas Meneghel Rodrigues , Higor Vieira Alves , Ramon de Carvalho Valle Subject: [PATCH] common_lib: Adding software_manager.py Date: Mon, 29 Nov 2010 00:44:52 -0200 Message-Id: <1290998692-19649-1-git-send-email-lmr@redhat.com> X-Scanned-By: MIMEDefang 2.68 on 10.5.11.23 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.3 (demeter1.kernel.org [140.211.167.41]); Mon, 29 Nov 2010 02:45:16 +0000 (UTC) diff --git a/client/common_lib/software_manager.py b/client/common_lib/software_manager.py new file mode 100755 index 0000000..f67f667 --- /dev/null +++ b/client/common_lib/software_manager.py @@ -0,0 +1,788 @@ +#!/usr/bin/python +""" +Software package management library. + +This is an abstraction layer on top of the existing distributions high level +package managers. It supports package operations useful for testing purposes, +and multiple high level package managers (here called backends). If you want +to make this lib to support your particular package manager/distro, please +implement the given backend class. + +@author: Higor Vieira Alves (halves@br.ibm.com) +@author: Lucas Meneghel Rodrigues (lmr@redhat.com) +@author: Ramon de Carvalho Valle (rcvalle@br.ibm.com) + +@copyright: IBM 2008-2009 +@copyright: Red Hat 2009-2010 +""" +import os, re, logging, ConfigParser, optparse, random, string +try: + import yum +except: + pass +import common +from autotest_lib.client.bin import os_dep, utils +from autotest_lib.client.common_lib import error +from autotest_lib.client.common_lib import logging_config, logging_manager + + +def generate_random_string(length): + """ + Return a random string using alphanumeric characters. + + @length: Length of the string that will be generated. + """ + r = random.SystemRandom() + str = "" + chars = string.letters + string.digits + while length > 0: + str += r.choice(chars) + length -= 1 + return str + + +class SoftwareManagerLoggingConfig(logging_config.LoggingConfig): + """ + Used with the sole purpose of providing convenient logging setup + for the KVM test auxiliary programs. + """ + def configure_logging(self, results_dir=None, verbose=False): + super(SoftwareManagerLoggingConfig, self).configure_logging( + use_console=True, + verbose=verbose) + + +class SystemInspector(object): + """ + System inspector class. + + This may grow up to include more complete reports of operating system and + machine properties. + """ + def __init__(self): + """ + Probe system, and save information for future reference. + """ + self.distro = utils.get_os_vendor() + self.high_level_pms = ['apt-get', 'yum', 'zypper'] + + + def get_package_management(self): + """ + Determine the supported package management systems present on the + system. If more than one package management system installed, try + to find the best supported system. + """ + list_supported = [] + for high_level_pm in self.high_level_pms: + try: + os_dep.command(high_level_pm) + list_supported.append(high_level_pm) + except: + pass + + pm_supported = None + if len(list_supported) == 0: + pm_supported = None + if len(list_supported) == 1: + pm_supported = list_supported[0] + elif len(list_supported) > 1: + if 'apt-get' in list_supported and self.distro in ['Debian', 'Ubuntu']: + pm_supported = 'apt-get' + elif 'yum' in list_supported and self.distro == 'Fedora': + pm_supported = 'yum' + else: + pm_supported = list_supported[0] + + logging.debug('Package Manager backend: %s' % pm_supported) + return pm_supported + + +class SoftwareManager(object): + """ + Package management abstraction layer. + + It supports a set of common package operations for testing purposes, and it + uses the concept of a backend, a helper class that implements the set of + operations of a given package management tool. + """ + def __init__(self): + """ + Class constructor. + + Determines the best supported package management system for the given + operating system running and initializes the appropriate backend. + """ + inspector = SystemInspector() + backend_type = inspector.get_package_management() + if backend_type == 'yum': + self.backend = YumBackend() + elif backend_type == 'zypper': + self.backend = ZypperBackend() + elif backend_type == 'apt-get': + self.backend = AptBackend() + else: + raise NotImplementedError('Unimplemented package management ' + 'system: %s.' % backend_type) + + + def check_installed(self, name, version=None, arch=None): + """ + Check whether a package is installed on this system. + + @param name: Package name. + @param version: Package version. + @param arch: Package architecture. + """ + return self.backend.check_installed(name, version, arch) + + + def list_all(self): + """ + List all installed packages. + """ + return self.backend.list_all() + + + def list_files(self, name): + """ + Get a list of all files installed by package [name]. + + @param name: Package name. + """ + return self.backend.list_files(name) + + + def install(self, name): + """ + Install package [name]. + + @param name: Package name. + """ + return self.backend.install(name) + + + def remove(self, name): + """ + Remove package [name]. + + @param name: Package name. + """ + return self.backend.remove(name) + + + def add_repo(self, url): + """ + Add package repo described by [url]. + + @param name: URL of the package repo. + """ + return self.backend.add_repo(url) + + + def remove_repo(self, url): + """ + Remove package repo described by [url]. + + @param url: URL of the package repo. + """ + return self.backend.remove_repo(url) + + + def upgrade(self): + """ + Upgrade all packages available. + """ + return self.backend.upgrade() + + + def provides(self, file): + """ + Returns a list of packages that provides a given capability to the + system (be it a binary, a library). + + @param file: Path to the file. + """ + return self.backend.provides(file) + + + def install_what_provides(self, file): + """ + Installs package that provides [file]. + + @param file: Path to file. + """ + provides = self.provides(file) + if provides is not None: + self.install(provides) + else: + logging.warning('No package seems to provide %s', file) + + +class RpmBackend(object): + """ + This class implements operations executed with the rpm package manager. + + rpm is a lower level package manager, used by higher level managers such + as yum and zypper. + """ + def __init__(self): + self.lowlevel_base_cmd = os_dep.command('rpm') + + + def _check_installed_version(self, name, version): + """ + Helper for the check_installed public method. + + @param name: Package name. + @param version: Package version. + """ + cmd = (self.lowlevel_base_cmd + ' -q --qf %{VERSION} ' + name + + ' 2> /dev/null') + inst_version = utils.system_output(cmd) + + if inst_version >= version: + return True + else: + return False + + + def check_installed(self, name, version=None, arch=None): + """ + Check if package [name] is installed. + + @param name: Package name. + @param version: Package version. + @param arch: Package architecture. + """ + if arch: + cmd = (self.lowlevel_base_cmd + ' -q --qf %{ARCH} ' + name + + ' 2> /dev/null') + inst_archs = utils.system_output(cmd) + inst_archs = inst_archs.split('\n') + + for inst_arch in inst_archs: + if inst_arch == arch: + return self._check_installed_version(name, version) + return False + + elif version: + return self._check_installed_version(name, version) + else: + cmd = 'rpm -q ' + name + ' 2> /dev/null' + return (os.system(cmd) == 0) + + + def list_all(self): + """ + List all installed packages. + """ + installed_packages = utils.system_output('rpm -qa').splitlines() + return installed_packages + + + def list_files(self, name): + """ + List files installed on the system by package [name]. + + @param name: Package name. + """ + path = os.path.abspath(name) + if os.path.isfile(path): + option = '-qlp' + name = path + else: + option = '-ql' + + l_cmd = 'rpm' + ' ' + option + ' ' + name + ' 2> /dev/null' + + try: + result = utils.system_output(l_cmd) + list_files = result.split('\n') + return list_files + except error.CmdError: + return [] + + +class DpkgBackend(object): + """ + This class implements operations executed with the dpkg package manager. + + dpkg is a lower level package manager, used by higher level managers such + as apt and aptitude. + """ + def __init__(self): + self.lowlevel_base_cmd = os_dep.command('dpkg') + + + def check_installed(self, name): + if os.path.isfile(name): + n_cmd = (self.lowlevel_base_cmd + ' -f ' + name + + ' Package 2>/dev/null') + name = utils.system_output(n_cmd) + i_cmd = self.lowlevel_base_cmd + ' -s ' + name + ' 2>/dev/null' + # Checking if package is installed + package_status = utils.system_output(i_cmd, ignore_status=True) + not_inst_pattern = re.compile('not-installed', re.IGNORECASE) + dpkg_not_installed = re.search(not_inst_pattern, package_status) + if dpkg_not_installed: + return False + return True + + + def list_all(self): + """ + List all packages available in the system. + """ + installed_packages = [] + raw_list = utils.system_output('dpkg -l').splitlines()[5:] + for line in raw_list: + parts = line.split() + if parts[0] == "ii": # only grab "installed" packages + installed_packages.append("%s-%s" % (parts[1], parts[2])) + + + def list_files(self, package): + """ + List files installed by package [package]. + + @param package: Package name. + @return: List of paths installed by package. + """ + if os.path.isfile(package): + l_cmd = self.lowlevel_base_cmd + ' -c ' + package + else: + l_cmd = self.lowlevel_base_cmd + ' -l ' + package + return utils.system_output(l_cmd).split('\n') + + +class YumBackend(RpmBackend): + """ + Implements the yum backend for software manager. + + Set of operations for the yum package manager, commonly found on Yellow Dog + Linux and Red Hat based distributions, such as Fedora and Red Hat + Enterprise Linux. + """ + def __init__(self): + """ + Initializes the base command and the yum package repository. + """ + super(YumBackend, self).__init__() + executable = os_dep.command('yum') + base_arguments = '-y' + self.base_command = executable + ' ' + base_arguments + self.repo_file_path = '/etc/yum.repos.d/autotest.repo' + self.cfgparser = ConfigParser.ConfigParser() + self.cfgparser.read(self.repo_file_path) + y_cmd = executable + ' --version | head -1' + self.yum_version = utils.system_output(y_cmd, ignore_status=True) + logging.debug('Yum backend initialized') + logging.debug('Yum version: %s' % self.yum_version) + self.yum_base = yum.YumBase() + + + def _cleanup(self): + """ + Clean up the yum cache so new package information can be downloaded. + """ + utils.system("yum clean all") + + + def install(self, name): + """ + Installs package [name]. Handles local installs. + """ + if os.path.isfile(name): + name = os.path.abspath(name) + command = 'localinstall' + else: + command = 'install' + + i_cmd = self.base_command + ' ' + command + ' ' + name + + try: + utils.system(i_cmd) + return True + except: + return False + + + def remove(self, name): + """ + Removes package [name]. + + @param name: Package name (eg. 'ipython'). + """ + r_cmd = self.base_command + ' ' + 'erase' + ' ' + name + try: + utils.system(r_cmd) + return True + except: + return False + + + def add_repo(self, url): + """ + Adds package repository located on [url]. + + @param url: Universal Resource Locator of the repository. + """ + # Check if we URL is already set + for section in self.cfgparser.sections(): + for option, value in self.cfgparser.items(section): + if option == 'url' and value == url: + return True + + # Didn't find it, let's set it up + while True: + section_name = 'software_manager' + '_' + generate_random_string(4) + if not self.cfgparser.has_section(section_name): + break + self.cfgparser.add_section(section_name) + self.cfgparser.set(section_name, 'name', + 'Repository added by the autotest software manager.') + self.cfgparser.set(section_name, 'url', url) + self.cfgparser.set(section_name, 'enabled', 1) + self.cfgparser.set(section_name, 'gpgcheck', 0) + self.cfgparser.write(self.repo_file_path) + + + def remove_repo(self, url): + """ + Removes package repository located on [url]. + + @param url: Universal Resource Locator of the repository. + """ + for section in self.cfgparser.sections(): + for option, value in self.cfgparser.items(section): + if option == 'url' and value == url: + self.cfgparser.remove_section(section) + self.cfgparser.write(self.repo_file_path) + + + def upgrade(self): + """ + Upgrade all available packages. + """ + r_cmd = self.base_command + ' ' + 'update' + try: + utils.system(r_cmd) + return True + except: + return False + + + def provides(self, name): + """ + Returns a list of packages that provides a given capability. + + @param name: Capability name (eg, 'foo'). + """ + d_provides = self.yum_base.searchPackageProvides(args=[name]) + provides_list = [key for key in d_provides] + if provides_list: + logging.info("Package %s provides %s", provides_list[0], name) + return str(provides_list[0]) + else: + return None + + +class ZypperBackend(RpmBackend): + """ + Implements the zypper backend for software manager. + + Set of operations for the zypper package manager, found on SUSE Linux. + """ + def __init__(self): + """ + Initializes the base command and the yum package repository. + """ + super(ZypperBackend, self).__init__() + self.base_command = os_dep.command('zypper') + ' -n' + z_cmd = self.base_command + ' --version' + self.zypper_version = utils.system_output(z_cmd, ignore_status=True) + logging.debug('Zypper backend initialized') + logging.debug('Zypper version: %s' % self.zypper_version) + + + def install(self, name): + """ + Installs package [name]. Handles local installs. + + @param name: Package Name. + """ + path = os.path.abspath(name) + i_cmd = self.base_command + ' install -l ' + name + try: + utils.system(i_cmd) + return True + except: + return False + + + def add_repo(self, url): + """ + Adds repository [url]. + + @param url: URL for the package repository. + """ + ar_cmd = self.base_command + ' addrepo ' + url + try: + utils.system(ar_cmd) + return True + except: + return False + + + def remove_repo(self, url): + """ + Removes repository [url]. + + @param url: URL for the package repository. + """ + rr_cmd = self.base_command + ' removerepo ' + url + try: + utils.system(rr_cmd) + return True + except: + return False + + + def remove(self, name): + """ + Removes package [name]. + """ + r_cmd = self.base_command + ' ' + 'erase' + ' ' + name + + try: + utils.system(r_cmd) + return True + except: + return False + + + def upgrade(self): + """ + Upgrades all packages of the system. + """ + u_cmd = self.base_command + ' update -l' + + try: + utils.system(u_cmd) + return True + except: + return False + + + def provides(self, name): + """ + Searches for what provides a given file. + + @param name: File path. + """ + p_cmd = self.base_command + ' what-provides ' + name + list_provides = [] + try: + p_output = utils.system_output(p_cmd).split('\n')[4:] + for line in p_output: + line = [a.strip() for a in line.split('|')] + try: + state, pname, type, version, arch, repository = line + if pname not in list_provides: + list_provides.append(pname) + except IndexError: + pass + if len(list_provides) > 1: + logging.warning('More than one package found, ' + 'opting by the first queue result') + if list_provides: + logging.info("Package %s provides %s", list_provides[0], name) + return list_provides[0] + return None + except: + return None + + +class AptBackend(DpkgBackend): + """ + Implements the apt backend for software manager. + + Set of operations for the apt package manager, commonly found on Debian and + Debian based distributions, such as Ubuntu Linux. + """ + def __init__(self): + """ + Initializes the base command and the debian package repository. + """ + super(AptBackend, self).__init__() + executable = os_dep.command('apt-get') + self.base_command = executable + ' -y' + self.repo_file_path = '/etc/apt/sources.list.d/autotest' + self.apt_version = utils.system_output('apt-get -v | head -1', + ignore_status=True) + logging.debug('Apt backend initialized') + logging.debug('apt version: %s' % self.apt_version) + + + def install(self, name): + """ + Installs package [name]. + + @param name: Package name. + """ + command = 'install' + i_cmd = self.base_command + ' ' + command + ' ' + name + + try: + utils.system(i_cmd) + return True + except: + return False + + + def remove(self, name): + """ + Remove package [name]. + + @param name: Package name. + """ + command = 'remove' + flag = '--purge' + r_cmd = self.base_command + ' ' + command + ' ' + flag + ' ' + name + + try: + utils.system(r_cmd) + return True + except: + return False + + + def add_repo(self, repo): + """ + Add an apt repository. + + @param repo: Repository string. Example: + 'deb http://archive.ubuntu.com/ubuntu/ maverick universe' + """ + repo_file = open(self.repo_file_path, 'a') + repo_file_contents = repo_file.read() + if repo not in repo_file_contents: + repo_file.write(repo) + + + def remove_repo(self, repo): + """ + Remove an apt repository. + + @param repo: Repository string. Example: + 'deb http://archive.ubuntu.com/ubuntu/ maverick universe' + """ + repo_file = open(self.repo_file_path, 'r') + new_file_contents = [] + for line in repo_file.readlines: + if not line == repo: + new_file_contents.append(line) + repo_file.close() + new_file_contents = "\n".join(new_file_contents) + repo_file.open(self.repo_file_path, 'w') + repo_file.write(new_file_contents) + repo_file.close() + + + def upgrade(self): + """ + Upgrade all packages of the system with eventual new versions. + """ + ud_command = 'update' + ud_cmd = self.base_command + ' ' + ud_command + try: + utils.system(ud_cmd) + except: + logging.error("Apt package update failed") + up_command = 'upgrade' + up_cmd = self.base_command + ' ' + up_command + try: + utils.system(up_cmd) + return True + except: + return False + + + def provides(self, file): + """ + Return a list of packages that provide [file]. + + @param file: File path. + """ + if not self.check_installed('apt-file'): + self.install('apt-file') + command = os_dep.command('apt-file') + cache_update_cmd = command + ' update' + try: + utils.system(cache_update_cmd, ignore_status=True) + except: + logging.error("Apt file cache update failed") + fu_cmd = command + ' search ' + file + try: + provides = utils.system_output(fu_cmd).split('\n') + list_provides = [] + for line in provides: + if line: + try: + line = line.split(':') + package = line[0].strip() + path = line[1].strip() + if path == file and package not in list_provides: + list_provides.append(package) + except IndexError: + pass + if len(list_provides) > 1: + logging.warning('More than one package found, ' + 'opting by the first queue result') + if list_provides: + logging.info("Package %s provides %s", list_provides[0], file) + return list_provides[0] + return None + except: + return None + + +if __name__ == '__main__': + parser = optparse.OptionParser( + "usage: %prog [install|remove|list-all|list-files|add-repo|remove-repo|" + "upgrade|what-provides|install-what-provides] arguments") + parser.add_option('--verbose', dest="debug", action='store_true', + help='include debug messages in console output') + + options, args = parser.parse_args() + debug = options.debug + logging_manager.configure_logging(SoftwareManagerLoggingConfig(), + verbose=debug) + software_manager = SoftwareManager() + if args: + action = args[0] + args = " ".join(args[1:]) + else: + action = 'show-help' + + if action == 'install': + software_manager.install(args) + elif action == 'remove': + software_manager.remove(args) + if action == 'list-all': + software_manager.list_all() + elif action == 'list-files': + software_manager.list_files(args) + elif action == 'add-repo': + software_manager.add_repo(args) + elif action == 'remove-repo': + software_manager.remove_repo(args) + elif action == 'upgrade': + software_manager.upgrade() + elif action == 'what-provides': + software_manager.provides(args) + elif action == 'install-what-provides': + software_manager.install_what_provides(args) + elif action == 'show-help': + parser.print_help()