@@ -6,7 +6,7 @@ KVM test utility functions.
import time, string, random, socket, os, signal, re, logging, commands, cPickle
import fcntl, shelve, ConfigParser, threading, sys, UserDict, inspect, tarfile
-import struct
+import struct, shutil
from autotest_lib.client.bin import utils, os_dep
from autotest_lib.client.common_lib import error, logging_config
import rss_client, aexpect
@@ -2110,7 +2110,7 @@ def get_default_koji_tag():
return DEFAULT_KOJI_TAG
-class KojiPkgSpec:
+class KojiPkgSpec(object):
'''
A package specification syntax parser for Koji
@@ -2405,6 +2405,698 @@ def mount(src, mount_point, type, perm="rw"):
return False
+class GitRepoHelper(object):
+ '''
+ Helps to deal with git repos, mostly fetching content from a repo
+ '''
+ def __init__(self, uri, branch, destination_dir, commit=None, lbranch=None):
+ '''
+ Instantiates a new GitRepoHelper
+
+ @type uri: string
+ @param uri: git repository url
+ @type branch: string
+ @param branch: git remote branch
+ @type destination_dir: string
+ @param destination_dir: path of a dir where to save downloaded code
+ @type commit: string
+ @param commit: specific commit to download
+ @type lbranch: string
+ @param lbranch: git local branch name, if different from remote
+ '''
+ self.uri = uri
+ self.branch = branch
+ self.destination_dir = destination_dir
+ self.commit = commit
+ if self.lbranch is None:
+ self.lbranch = branch
+
+
+ def init(self):
+ '''
+ Initializes a directory for receiving a verbatim copy of git repo
+
+ This creates a directory if necessary, and either resets or inits
+ the repo
+ '''
+ if not os.path.exists(self.destination_dir):
+ logging.debug('Creating directory %s for git repo %s',
+ self.destination_dir, self.uri)
+ os.makedirs(self.destination_dir)
+
+ os.chdir(self.destination_dir)
+
+ if os.path.exists('.git'):
+ logging.debug('Resetting previously existing git repo at %s for '
+ 'receiving git repo %s',
+ self.destination_dir, self.uri)
+ utils.system('git reset --hard')
+ else:
+ logging.debug('Initializing new git repo at %s for receiving '
+ 'git repo %s',
+ self.destination_dir, self.uri)
+ utils.system('git init')
+
+
+ def fetch(self):
+ '''
+ Performs a git fetch from the remote repo
+ '''
+ logging.info("Fetching git [REP '%s' BRANCH '%s'] -> %s",
+ self.uri, self.branch, self.destination_dir)
+ os.chdir(self.destination_dir)
+ utils.system("git fetch -q -f -u -t %s %s:%s" % (self.uri,
+ self.branch,
+ self.lbranch))
+
+
+ def checkout(self):
+ '''
+ Performs a git checkout for a given branch and start point (commit)
+ '''
+ os.chdir(self.destination_dir)
+
+ logging.debug('Checking out local branch %s', self.lbranch)
+ utils.system("git checkout %s" % self.lbranch)
+
+ if self.commit is not None:
+ logging.debug('Checking out commit %s', self.commit)
+ utils.system("git checkout %s" % self.commit)
+
+ h = utils.system_output('git log --pretty=format:"%H" -1').strip()
+ try:
+ desc = "tag %s" % utils.system_output("git describe")
+ except error.CmdError:
+ desc = "no tag found"
+
+ logging.info("Commit hash for %s is %s (%s)", self.name, h, desc)
+
+
+ def execute(self):
+ '''
+ Performs all steps necessary to initialize and download a git repo
+
+ This includes the init, fetch and checkout steps in one single
+ utility method.
+ '''
+ self.init()
+ self.fetch()
+ self.checkout()
+
+
+class GitRepoParamHelper(GitRepoHelper):
+ '''
+ Helps to deal with git repos specified in cartersian config files
+
+ This class attempts to make it simple to manage a git repo, by using a
+ naming standard that follows this basic syntax:
+
+ <prefix>_name_<suffix>
+
+ <prefix> is always 'git_repo' and <suffix> sets options for this git repo.
+ Example for repo named foo:
+
+ git_repo_foo_uri = git://git.foo.org/foo.git
+ git_repo_foo_branch = master
+ git_repo_foo_lbranch = master
+ git_repo_foo_commit = bb5fb8e678aabe286e74c4f2993dc2a9e550b627
+ '''
+ def __init__(self, params, name, destination_dir):
+ '''
+ Instantiates a new GitRepoParamHelper
+ '''
+ self.params = params
+ self.name = name
+ self.destination_dir = destination_dir
+ self._parse_params()
+
+
+ def _parse_params(self):
+ '''
+ Parses the params items for entries related to this repo
+
+ This method currently does everything that the parent class __init__()
+ method does, that is, sets all instance variables needed by other
+ methods. That means it's not strictly necessary to call parent's
+ __init__().
+ '''
+ config_prefix = 'git_repo_%s' % self.name
+ logging.debug('Parsing parameters for git repo %s, configuration '
+ 'prefix is %s' % (self.name, config_prefix))
+
+ self.uri = self.params.get('%s_uri' % config_prefix)
+ logging.debug('Git repo %s uri: %s' % (self.name, self.uri))
+
+ self.branch = self.params.get('%s_branch' % config_prefix, 'master')
+ logging.debug('Git repo %s branch: %s' % (self.name, self.branch))
+
+ self.lbranch = self.params.get('%s_lbranch' % config_prefix)
+ if self.lbranch is None:
+ self.lbranch = self.branch
+ logging.debug('Git repo %s lbranch: %s' % (self.name, self.lbranch))
+
+ self.commit = self.params.get('%s_commit' % config_prefix)
+ if self.commit is None:
+ logging.debug('Git repo %s commit is not set' % self.name)
+ else:
+ logging.debug('Git repo %s commit: %s' % (self.name, self.commit))
+
+
+class LocalSourceDirHelper(object):
+ '''
+ Helper class to deal with source code sitting somewhere in the filesystem
+ '''
+ def __init__(self, source_dir, destination_dir):
+ '''
+ @param source_dir:
+ @param destination_dir:
+ @return: new LocalSourceDirHelper instance
+ '''
+ self.source = source_dir
+ self.destination = destination_dir
+
+
+ def execute(self):
+ '''
+ Copies the source directory to the destination directory
+ '''
+ if os.path.isdir(self.destination):
+ shutil.rmtree(self.destination)
+
+ if os.path.isdir(self.source):
+ shutil.copytree(self.source, self.destination)
+
+
+class LocalSourceDirParamHelper(LocalSourceDirHelper):
+ '''
+ Helps to deal with source dirs specified in cartersian config files
+
+ This class attempts to make it simple to manage a source dir, by using a
+ naming standard that follows this basic syntax:
+
+ <prefix>_name_<suffix>
+
+ <prefix> is always 'local_src' and <suffix> sets options for this source
+ dir. Example for source dir named foo:
+
+ local_src_foo_path = /home/user/foo
+ '''
+ def __init__(self, params, name, destination_dir):
+ '''
+ Instantiate a new LocalSourceDirParamHelper
+ '''
+ self.params = params
+ self.name = name
+ self.destination_dir = destination_dir
+ self._parse_params()
+
+
+ def _parse_params(self):
+ '''
+ Parses the params items for entries related to source dir
+ '''
+ config_prefix = 'local_src_%s' % self.name
+ logging.debug('Parsing parameters for local source %s, configuration '
+ 'prefix is %s' % (self.name, config_prefix))
+
+ self.path = self.params.get('%s_path' % config_prefix)
+ logging.debug('Local source directory %s path: %s' % (self.name,
+ self.path))
+ self.source = self.path
+ self.destination = self.destination_dir
+
+
+class LocalTarHelper(object):
+ '''
+ Helper class to deal with source code in a local tarball
+ '''
+ def __init__(self, source_file, destination_dir):
+ self.source = source
+ self.destination = destination_dir
+
+
+ def extract(self):
+ '''
+ Extracts the tarball into the destination directory
+ '''
+ if os.path.isdir(self.destination):
+ shutil.rmtree(self.destination)
+
+ if os.path.isfile(self.source) and tarfile.is_tarfile(self.source):
+
+ name = os.path.basename(self.destination)
+ temp_dir = os.path.join(os.path.dirname(self.destination),
+ '%s.tmp' % name)
+ logging.debug('Temporary directory for extracting tarball is %s' %
+ temp_dir)
+
+ if not os.path.isdir(temp_dir):
+ os.makedirs(temp_dir)
+
+ tarball = tarfile.open(self.source)
+ tarball.extractall(temp_dir)
+
+ #
+ # If there's a directory at the toplevel of the tarfile, assume
+ # it's the root for the contents, usually source code
+ #
+ tarball_info = tarball.members[0]
+ if tarball_info.isdir():
+ content_path = os.path.join(temp_dir,
+ tarball_info.name)
+ else:
+ content_path = temp_dir
+
+ #
+ # Now move the content directory to the final destination
+ #
+ shutil.move(content_path, self.destination)
+
+ else:
+ raise OSError("%s is not a file or tar file" % self.source)
+
+
+ def execute(self):
+ '''
+ Executes all action this helper is suposed to perform
+
+ This is the main entry point method for this class, and all other
+ helper classes.
+ '''
+ self.extract()
+
+
+class LocalTarParamHelper(LocalTarHelper):
+ '''
+ Helps to deal with source tarballs specified in cartersian config files
+
+ This class attempts to make it simple to manage a tarball with source code,
+ by using a naming standard that follows this basic syntax:
+
+ <prefix>_name_<suffix>
+
+ <prefix> is always 'local_tar' and <suffix> sets options for this source
+ tarball. Example for source tarball named foo:
+
+ local_tar_foo_path = /tmp/foo-1.0.tar.gz
+ '''
+ def __init__(self, params, name, destination_dir):
+ '''
+ Instantiates a new LocalTarParamHelper
+ '''
+ self.params = params
+ self.name = name
+ self.destination_dir = destination_dir
+ self._parse_params()
+
+
+ def _parse_params(self):
+ '''
+ Parses the params items for entries related to this local tar helper
+ '''
+ config_prefix = 'local_tar_%s' % self.name
+ logging.debug('Parsing parameters for local tar %s, configuration '
+ 'prefix is %s' % (self.name, config_prefix))
+
+ self.path = self.params.get('%s_path' % config_prefix)
+ logging.debug('Local source tar %s path: %s' % (self.name,
+ self.path))
+ self.source = self.path
+ self.destination = self.destination_dir
+
+
+class RemoteTarHelper(LocalTarHelper):
+ '''
+ Helper that fetches a tarball and extracts it locally
+ '''
+ def __init__(self, source_uri, destination_dir):
+ self.source = source_uri
+ self.destination = destination_dir
+
+
+ def execute(self):
+ '''
+ Executes all action this helper class is suposed to perform
+
+ This is the main entry point method for this class, and all other
+ helper classes.
+
+ This implementation fetches the remote tar file and then extracts
+ it using the functionality present in the parent class.
+ '''
+ name = os.path.basename(self.source)
+ base_dest = os.path.dirname(self.destination_dir)
+ dest = os.path.join(base_dest, name)
+ utils.get_file(self.source, dest)
+ self.source = dest
+ self.extract()
+
+
+class RemoteTarParamHelper(RemoteTarHelper):
+ '''
+ Helps to deal with remote source tarballs specified in cartersian config
+
+ This class attempts to make it simple to manage a tarball with source code,
+ by using a naming standard that follows this basic syntax:
+
+ <prefix>_name_<suffix>
+
+ <prefix> is always 'local_tar' and <suffix> sets options for this source
+ tarball. Example for source tarball named foo:
+
+ remote_tar_foo_uri = http://foo.org/foo-1.0.tar.gz
+ '''
+ def __init__(self, params, name, destination_dir):
+ '''
+ Instantiates a new RemoteTarParamHelper instance
+ '''
+ self.params = params
+ self.name = name
+ self.destination_dir = destination_dir
+ self._parse_params()
+
+
+ def _parse_params(self):
+ '''
+ Parses the params items for entries related to this remote tar helper
+ '''
+ config_prefix = 'remote_tar_%s' % self.name
+ logging.debug('Parsing parameters for remote tar %s, configuration '
+ 'prefix is %s' % (self.name, config_prefix))
+
+ self.uri = self.params.get('%s_uri' % config_prefix)
+ logging.debug('Remote source tar %s uri: %s' % (self.name,
+ self.uri))
+ self.source = self.uri
+ self.destination = self.destination_dir
+
+
+class PatchHelper(object):
+ '''
+ Helper that encapsulates the patching of source code with patch files
+ '''
+ def __init__(self, source_dir, patches):
+ '''
+ Initializes a new PatchHelper
+ '''
+ self.source_dir = source_dir
+ self.patches = patches
+
+
+ def download(self):
+ '''
+ Copies patch files from remote locations to the source directory
+ '''
+ for patch in self.patches:
+ utils.get_file(patch, os.path.join(self.source_dir,
+ os.path.basename(patch)))
+
+
+ def patch(self):
+ '''
+ Patches the source dir with all patch files
+ '''
+ os.chdir(self.source_dir)
+ for patch in self.patches:
+ patch_file = os.path.join(self.source_dir,
+ os.path.basename(patch))
+ utils.system('patch -p1 < %s' % os.path.basename(patch))
+
+
+ def execute(self):
+ '''
+ Performs all steps necessary to download patches and apply them
+ '''
+ self.download()
+ self.patch()
+
+
+class PatchParamHelper(PatchHelper):
+ '''
+ Helps to deal with patches specified in cartersian config files
+
+ This class attempts to make it simple to patch source coude, by using a
+ naming standard that follows this basic syntax:
+
+ [<git_repo>|<local_src>|<local_tar>|<remote_tar>]_<name>_patches
+
+ <prefix> is either a 'local_src' or 'git_repo', that, together with <name>
+ specify a directory containing source code to receive the patches. That is,
+ for source code coming from git repo foo, patches would be specified as:
+
+ git_repo_foo_patches = ['http://foo/bar.patch', 'http://foo/baz.patch']
+
+ And for for patches to be applied on local source code named also foo:
+
+ local_src_foo_patches = ['http://foo/bar.patch', 'http://foo/baz.patch']
+ '''
+ def __init__(self, params, prefix, source_dir):
+ '''
+ Initializes a new PatchParamHelper instance
+ '''
+ self.params = params
+ self.prefix = prefix
+ self.source_dir = source_dir
+ self._parse_params()
+
+
+ def _parse_params(self):
+ '''
+ Parses the params items for entries related to this set of patches
+
+ This method currently does everything that the parent class __init__()
+ method does, that is, sets all instance variables needed by other
+ methods. That means it's not strictly necessary to call parent's
+ __init__().
+ '''
+ logging.debug('Parsing patch parameters for prefix %s' % self.prefix)
+ patches_param_key = '%s_patches' % self.prefix
+
+ self.patches_str = self.params.get(patches_param_key, '[]')
+ logging.debug('Patches config for prefix %s: %s' % (self.prefix,
+ self.patches_str))
+
+ self.patches = eval(self.patches_str)
+ logging.debug('Patches for prefix %s: %s' % (self.prefix,
+ ", ".join(self.patches)))
+
+
+class GnuSourceBuildInvalidSource(Exception):
+ '''
+ Exception raised when build source dir/file is not valid
+ '''
+ pass
+
+
+class GnuSourceBuildHelper(object):
+ '''
+ Handles software installation of GNU-like source code
+
+ This basically means that the build will go though the classic GNU
+ autotools steps: ./configure, make, make install
+ '''
+ def __init__(self, source, build_dir, prefix,
+ configure_options=[]):
+ '''
+ @type source: string
+ @param source: source directory or tarball
+ @type prefix: string
+ @param prefix: installation prefix
+ @type build_dir: string
+ @param build_dir: temporary directory used for building the source code
+ @type configure_options: list
+ @param configure_options: options to pass to configure
+ @throws: GnuSourceBuildInvalidSource
+ '''
+ self.source = source
+ self.build_dir = build_dir
+ self.prefix = prefix
+ self.configure_options = configure_options
+ self.include_pkg_config_path()
+
+
+ def include_pkg_config_path(self):
+ '''
+ Adds the current prefix to the list of paths that pkg-config searches
+
+ This is currently not optional as there is no observed adverse side
+ effects of enabling this. As the "prefix" is usually only valid during
+ a test run, we believe that having other pkg-config files (*.pc) in
+ either '<prefix>/share/pkgconfig' or '<prefix>/lib/pkgconfig' is
+ exactly for the purpose of using them.
+
+ @returns: None
+ '''
+ env_var = 'PKG_CONFIG_PATH'
+
+ include_paths = [os.path.join(self.prefix, 'share', 'pkgconfig'),
+ os.path.join(self.prefix, 'lib', 'pkgconfig')]
+
+ if os.environ.has_key(env_var):
+ paths = os.environ[env_var].split(':')
+ for include_path in include_paths:
+ if include_path not in paths:
+ paths.append(prefix_pkg_config)
+ os.environ[env_var] = ':'.join(paths)
+ else:
+ os.environ[env_var] = ':'.join(include_paths)
+
+ logging.debug('PKG_CONFIG_PATH is: %s' % os.environ['PKG_CONFIG_PATH'])
+
+
+ def get_configure_path(self):
+ '''
+ Checks if 'configure' exists, if not, return 'autogen.sh' as a fallback
+ '''
+ configure_path = os.path.abspath(os.path.join(self.source,
+ "configure"))
+ autogen_path = os.path.abspath(os.path.join(self.source,
+ "autogen.sh"))
+ if os.path.exists(configure_path):
+ return configure_path
+ elif os.path.exists(autogen_path):
+ return autogen_path
+ else:
+ raise GnuSourceBuildInvalidSource('configure script does not exist')
+
+
+ def get_available_configure_options(self):
+ '''
+ Return the list of available options of a GNU like configure script
+
+ This will run the "configure" script at the source directory
+
+ @returns: list of options accepted by configure script
+ '''
+ help_raw = utils.system_output('%s --help' % self.get_configure_path(),
+ ignore_status=True)
+ help_output = help_raw.split("\n")
+ option_list = []
+ for line in help_output:
+ cleaned_line = line.lstrip()
+ if cleaned_line.startswith("--"):
+ option = cleaned_line.split()[0]
+ option = option.split("=")[0]
+ option_list.append(option)
+
+ return option_list
+
+
+ def enable_debug_symbols(self):
+ '''
+ Enables option that leaves debug symbols on compiled software
+
+ This makes debugging a lot easier.
+ '''
+ enable_debug_option = "--disable-strip"
+ if enable_debug_option in self.get_available_configure_options():
+ self.configure_options.append(enable_debug_option)
+ logging.debug('Enabling debug symbols with option: %s' %
+ enable_debug_option)
+
+
+ def get_configure_command(self):
+ '''
+ Formats configure script with all options set
+
+ @returns: string with all configure options, including prefix
+ '''
+ prefix_option = "--prefix=%s" % self.prefix
+ options = self.configure_options
+ options.append(prefix_option)
+ return "%s %s" % (self.get_configure_path(),
+ " ".join(options))
+
+
+ def configure(self):
+ '''
+ Runs the "configure" script passing apropriate command line options
+ '''
+ configure_command = self.get_configure_command()
+ logging.info('Running configure on build dir')
+ os.chdir(self.build_dir)
+ utils.system(configure_command)
+
+
+ def make(self):
+ '''
+ Runs "make" using the correct number of parallel jobs
+ '''
+ parallel_make_jobs = utils.count_cpus()
+ make_command = "make -j %s" % parallel_make_jobs
+ logging.info("Running make on build dir")
+ os.chdir(self.build_dir)
+ utils.system(make_command)
+
+
+ def make_install(self):
+ '''
+ Runs "make install"
+ '''
+ os.chdir(self.build_dir)
+ utils.system("make install")
+
+
+ install = make_install
+
+
+ def execute(self):
+ '''
+ Runs appropriate steps for *building* this source code tree
+ '''
+ self.configure()
+ self.make()
+
+
+class GnuSourceBuildParamHelper(GnuSourceBuildHelper):
+ '''
+ Helps to deal with gnu_autotools build helper in cartersian config files
+
+ This class attempts to make it simple to build source coude, by using a
+ naming standard that follows this basic syntax:
+
+ [<git_repo>|<local_src>]_<name>_<option> = value
+
+ To pass extra options to the configure script, while building foo from a
+ git repo, set the following variable:
+
+ git_repo_foo_configure_options = --enable-feature
+ '''
+ def __init__(self, params, name, destination_dir, install_prefix):
+ '''
+ Instantiates a new GnuSourceBuildParamHelper
+ '''
+ self.params = params
+ self.name = name
+ self.destination_dir = destination_dir
+ self.install_prefix = install_prefix
+ self._parse_params()
+
+
+ def _parse_params(self):
+ '''
+ Parses the params items for entries related to source directory
+
+ This method currently does everything that the parent class __init__()
+ method does, that is, sets all instance variables needed by other
+ methods. That means it's not strictly necessary to call parent's
+ __init__().
+ '''
+ logging.debug('Parsing gnu_autotools build parameters for %s' %
+ self.name)
+
+ configure_opt_key = '%s_configure_options' % self.name
+ configure_options = self.params.get(configure_opt_key, '').split()
+ logging.debug('Configure options for %s: %s' % (self.name,
+ configure_options))
+
+ self.source = self.destination_dir
+ self.build_dir = self.destination_dir
+ self.prefix = self.install_prefix
+ self.configure_options = configure_options
+ self.include_pkg_config_path()
+
+
def install_host_kernel(job, params):
"""
Install a host kernel, given the appropriate params.