From patchwork Tue Sep 15 07:03:58 2009 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lucas Meneghel Rodrigues X-Patchwork-Id: 47590 Received: from vger.kernel.org (vger.kernel.org [209.132.176.167]) by demeter.kernel.org (8.14.2/8.14.2) with ESMTP id n8F74AHb004360 for ; Tue, 15 Sep 2009 07:04:10 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1758134AbZIOHED (ORCPT ); Tue, 15 Sep 2009 03:04:03 -0400 Received: (majordomo@vger.kernel.org) by vger.kernel.org id S1758133AbZIOHED (ORCPT ); Tue, 15 Sep 2009 03:04:03 -0400 Received: from mx1.redhat.com ([209.132.183.28]:23844 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1757991AbZIOHEA (ORCPT ); Tue, 15 Sep 2009 03:04:00 -0400 Received: from int-mx04.intmail.prod.int.phx2.redhat.com (int-mx04.intmail.prod.int.phx2.redhat.com [10.5.11.17]) by mx1.redhat.com (8.13.8/8.13.8) with ESMTP id n8F742rf007675; Tue, 15 Sep 2009 03:04:02 -0400 Received: from localhost.localdomain (vpn-12-102.rdu.redhat.com [10.11.12.102]) by int-mx04.intmail.prod.int.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id n8F73xfp010763; Tue, 15 Sep 2009 03:04:00 -0400 From: Lucas Meneghel Rodrigues To: autotest@test.kernel.org Cc: kvm@vger.kernel.org, Lucas Meneghel Rodrigues Subject: [PATCH 1/2] KVM test: Renaming some test files Date: Tue, 15 Sep 2009 04:03:58 -0300 Message-Id: <1252998239-4949-1-git-send-email-lmr@redhat.com> X-Scanned-By: MIMEDefang 2.67 on 10.5.11.17 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org Move kvm_guest_wizard.py to steps.py and kvm_install.py to build.py. Made the necessary arrangements in the subtest loading code. Signed-off-by: Lucas Meneghel Rodrigues --- client/tests/kvm/build.py | 333 ++++++++++++++++++++++++++++++++++ client/tests/kvm/control | 6 +- client/tests/kvm/kvm.py | 4 +- client/tests/kvm/kvm_guest_wizard.py | 249 ------------------------- client/tests/kvm/kvm_install.py | 333 ---------------------------------- client/tests/kvm/steps.py | 249 +++++++++++++++++++++++++ 6 files changed, 587 insertions(+), 587 deletions(-) create mode 100755 client/tests/kvm/build.py delete mode 100644 client/tests/kvm/kvm_guest_wizard.py delete mode 100755 client/tests/kvm/kvm_install.py create mode 100644 client/tests/kvm/steps.py diff --git a/client/tests/kvm/build.py b/client/tests/kvm/build.py new file mode 100755 index 0000000..9ad0e92 --- /dev/null +++ b/client/tests/kvm/build.py @@ -0,0 +1,333 @@ +import time, os, sys, urllib, re, signal, logging, datetime +from autotest_lib.client.bin import utils, test +from autotest_lib.client.common_lib import error +import kvm_utils + + +def load_kvm_modules(module_dir): + """ + Unload previously loaded kvm modules, then load modules present on any + sub directory of module_dir. Function will walk through module_dir until + it finds the modules. + + @param module_dir: Directory where the KVM modules are located. + """ + vendor = "intel" + if os.system("grep vmx /proc/cpuinfo 1>/dev/null") != 0: + vendor = "amd" + logging.debug("Detected CPU vendor as '%s'" %(vendor)) + + logging.debug("Killing any qemu processes that might be left behind") + utils.system("pkill qemu", ignore_status=True) + + logging.info("Unloading previously loaded KVM modules") + kvm_utils.unload_module("kvm") + if utils.module_is_loaded("kvm"): + message = "Failed to remove old KVM modules" + logging.error(message) + raise error.TestError(message) + + logging.info("Loading new KVM modules...") + kvm_module_path = None + kvm_vendor_module_path = None + # Search for the built KVM modules + for folder, subdirs, files in os.walk(module_dir): + if "kvm.ko" in files: + kvm_module_path = os.path.join(folder, "kvm.ko") + kvm_vendor_module_path = os.path.join(folder, "kvm-%s.ko" % vendor) + abort = False + if not kvm_module_path: + logging.error("Need a directory containing both kernel module and " + "userspace sources.") + logging.error("If you are trying to build only KVM userspace and use " + "the KVM modules you have already loaded, put " + "'load_modules': 'no' on the control file 'params' " + "dictionary.") + raise error.TestError("Could not find a built kvm.ko module on the " + "source dir.") + elif not os.path.isfile(kvm_vendor_module_path): + logging.error("Could not find KVM (%s) module that was supposed to be" + " built on the source dir", vendor) + abort = True + if abort: + raise error.TestError("Could not load KVM modules.") + utils.system("/sbin/insmod %s" % kvm_module_path) + time.sleep(1) + utils.system("/sbin/insmod %s" % kvm_vendor_module_path) + + if not utils.module_is_loaded("kvm"): + message = "Failed to load the KVM modules built for the test" + logging.error(message) + raise error.TestError(message) + + +def create_symlinks(test_bindir, prefix): + """ + Create symbolic links for the appropriate qemu and qemu-img commands on + the kvm test bindir. + + @param test_bindir: KVM test bindir + @param prefix: KVM prefix path + """ + qemu_path = os.path.join(test_bindir, "qemu") + qemu_img_path = os.path.join(test_bindir, "qemu-img") + if os.path.lexists(qemu_path): + os.unlink(qemu_path) + if os.path.lexists(qemu_img_path): + os.unlink(qemu_img_path) + kvm_qemu = os.path.join(prefix, "bin", "qemu-system-x86_64") + if not os.path.isfile(kvm_qemu): + raise error.TestError('Invalid qemu path') + kvm_qemu_img = os.path.join(prefix, "bin", "qemu-img") + if not os.path.isfile(kvm_qemu_img): + raise error.TestError('Invalid qemu-img path') + os.symlink(kvm_qemu, qemu_path) + os.symlink(kvm_qemu_img, qemu_img_path) + + +class SourceDirInstaller: + """ + Class that handles building/installing KVM directly from a tarball or + a single source code dir. + """ + def __init__(self, test, params): + """ + Initializes class attributes, and retrieves KVM code. + + @param test: kvm test object + @param params: Dictionary with test arguments + """ + install_mode = params["mode"] + srcdir = params.get("srcdir") + # KVM build prefix + self.test_bindir = test.bindir + prefix = os.path.join(test.bindir, 'build') + self.prefix = os.path.abspath(prefix) + # Are we going to load modules? + load_modules = params.get('load_modules') + if not load_modules: + self.load_modules = True + elif load_modules == 'yes': + self.load_modules = True + elif load_modules == 'no': + self.load_modules = False + + if install_mode == 'localsrc': + if not srcdir: + raise error.TestError("Install from source directory specified" + "but no source directory provided on the" + "control file.") + else: + self.srcdir = srcdir + self.repo_type = kvm_utils.check_kvm_source_dir(self.srcdir) + return + else: + srcdir = test.srcdir + if not os.path.isdir(srcdir): + os.makedirs(srcdir) + + if install_mode == 'release': + release_tag = params.get("release_tag") + release_dir = params.get("release_dir") + logging.info("Installing KVM from release tarball") + if not release_tag: + release_tag = kvm_utils.get_latest_kvm_release_tag(release_dir) + tarball = os.path.join(release_dir, "kvm-%s.tar.gz" % release_tag) + logging.info("Retrieving release kvm-%s" % release_tag) + tarball = utils.unmap_url("/", tarball, "/tmp") + + elif install_mode == 'snapshot': + logging.info("Installing KVM from snapshot") + snapshot_dir = params.get("snapshot_dir") + if not snapshot_dir: + raise error.TestError("Snapshot dir not provided") + snapshot_date = params.get("snapshot_date") + if not snapshot_date: + # Take yesterday's snapshot + d = (datetime.date.today() - + datetime.timedelta(1)).strftime("%Y%m%d") + else: + d = snapshot_date + tarball = os.path.join(snapshot_dir, "kvm-snapshot-%s.tar.gz" % d) + logging.info("Retrieving kvm-snapshot-%s" % d) + tarball = utils.unmap_url("/", tarball, "/tmp") + + elif install_mode == 'localtar': + tarball = params.get("tarball") + if not tarball: + raise error.TestError("KVM Tarball install specified but no" + " tarball provided on control file.") + logging.info("Installing KVM from a local tarball") + logging.info("Using tarball %s") + tarball = utils.unmap_url("/", params.get("tarball"), "/tmp") + + os.chdir(srcdir) + self.srcdir = os.path.join(srcdir, utils.extract_tarball(tarball)) + self.repo_type = kvm_utils.check_kvm_source_dir(self.srcdir) + + + def __build(self): + os.chdir(self.srcdir) + # For testing purposes, it's better to build qemu binaries with + # debugging symbols, so we can extract more meaningful stack traces. + cfg = "./configure --disable-strip --prefix=%s" % self.prefix + steps = [cfg, "make clean", "make -j %s" % (utils.count_cpus() + 1)] + logging.info("Building KVM") + for step in steps: + utils.system(step) + + + def __install(self): + os.chdir(self.srcdir) + logging.info("Installing KVM userspace") + if self.repo_type == 1: + utils.system("make -C qemu install") + elif self.repo_type == 2: + utils.system("make install") + create_symlinks(self.test_bindir, self.prefix) + + + def __load_modules(self): + load_kvm_modules(self.srcdir) + + + def install(self): + self.__build() + self.__install() + if self.load_modules: + self.__load_modules() + + +class GitInstaller: + def __init__(self, test, params): + """ + Initialize class parameters and retrieves code from git repositories. + + @param test: kvm test object. + @param params: Dictionary with test parameters. + """ + install_mode = params["mode"] + srcdir = params.get("srcdir", test.bindir) + if not srcdir: + os.makedirs(srcdir) + self.srcdir = srcdir + # KVM build prefix + self.test_bindir = test.bindir + prefix = os.path.join(test.bindir, 'build') + self.prefix = os.path.abspath(prefix) + # Are we going to load modules? + load_modules = params.get('load_modules') + if not load_modules: + self.load_modules = True + elif load_modules == 'yes': + self.load_modules = True + elif load_modules == 'no': + self.load_modules = False + + kernel_repo = params.get("git_repo") + user_repo = params.get("user_git_repo") + kmod_repo = params.get("kmod_repo") + + branch = params.get("git_branch", "master") + lbranch = params.get("lbranch") + + tag = params.get("git_tag", "HEAD") + user_tag = params.get("user_git_tag", "HEAD") + kmod_tag = params.get("kmod_git_tag", "HEAD") + + if not kernel_repo: + message = "KVM git repository path not specified" + logging.error(message) + raise error.TestError(message) + if not user_repo: + message = "KVM user git repository path not specified" + logging.error(message) + raise error.TestError(message) + + kernel_srcdir = os.path.join(srcdir, "kvm") + kvm_utils.get_git_branch(kernel_repo, branch, kernel_srcdir, tag, + lbranch) + self.kernel_srcdir = kernel_srcdir + + userspace_srcdir = os.path.join(srcdir, "kvm_userspace") + kvm_utils.get_git_branch(user_repo, branch, userspace_srcdir, user_tag, + lbranch) + self.userspace_srcdir = userspace_srcdir + + if kmod_repo: + kmod_srcdir = os.path.join (srcdir, "kvm_kmod") + kvm_utils.get_git_branch(kmod_repo, branch, kmod_srcdir, user_tag, + lbranch) + self.kmod_srcdir = kmod_srcdir + + + def __build(self): + if self.kmod_srcdir: + logging.info('Building KVM modules') + os.chdir(self.kmod_srcdir) + utils.system('./configure') + utils.system('make clean') + utils.system('make sync LINUX=%s' % self.kernel_srcdir) + utils.system('make -j %s' % utils.count_cpus()) + logging.info('Building KVM userspace code') + os.chdir(self.userspace_srcdir) + utils.system('./configure --disable-strip --prefix=%s' % + self.prefix) + utils.system('make clean') + utils.system('make -j %s' % utils.count_cpus()) + else: + os.chdir(self.userspace_srcdir) + utils.system('./configure --disable-strip --prefix=%s' % + self.prefix) + logging.info('Building KVM modules') + utils.system('make clean') + utils.system('make -C kernel LINUX=%s sync' % self.kernel_srcdir) + logging.info('Building KVM userspace code') + utils.system('make -j %s' % utils.count_cpus()) + + + def __install(self): + os.chdir(self.userspace_srcdir) + utils.system('make install') + create_symlinks(self.test_bindir, self.prefix) + + + def __load_modules(self): + if self.kmod_srcdir: + load_kvm_modules(self.kmod_srcdir) + else: + load_kvm_modules(self.userspace_srcdir) + + + def install(self): + self.__build() + self.__install() + if self.load_modules: + self.__load_modules() + + +def run_build(test, params, env): + """ + Installs KVM using the selected install mode. Most install methods will + take kvm source code, build it and install it to a given location. + + @param test: kvm test object. + @param params: Dictionary with test parameters. + @param env: Test environment. + """ + install_mode = params.get("mode") + srcdir = params.get("srcdir", test.srcdir) + params["srcdir"] = srcdir + + if install_mode == 'noinstall': + logging.info("Skipping installation") + return + elif install_mode in ['localsrc', 'localtar', 'release', 'snapshot']: + installer = SourceDirInstaller(test, params) + elif install_mode == 'git': + installer = GitInstaller(test, params) + else: + raise error.TestError('Invalid or unsupported' + ' install mode: %s' % install_mode) + + installer.install() diff --git a/client/tests/kvm/control b/client/tests/kvm/control index f8390d6..5e8d857 100644 --- a/client/tests/kvm/control +++ b/client/tests/kvm/control @@ -72,9 +72,9 @@ link_if_not_exist(pwd, qemu_img, 'qemu-img') # Params that will be passed to the KVM install/build test # -------------------------------------------------------- params = { - "name": "kvm_install", - "shortname": "kvm_install", - "type": "kvm_install", + "name": "build", + "shortname": "build", + "type": "build", "mode": "release", #"mode": "snapshot", #"mode": "localtar", diff --git a/client/tests/kvm/kvm.py b/client/tests/kvm/kvm.py index 8e66fc0..69d08b5 100644 --- a/client/tests/kvm/kvm.py +++ b/client/tests/kvm/kvm.py @@ -48,14 +48,14 @@ class kvm(test.test): # of the 'type' field self.test_routines = { # type module name routine - "steps": test_routine("kvm_guest_wizard", "run_steps"), + "build": test_routine("build", "run_build"), + "steps": test_routine("steps", "run_steps"), "stepmaker": test_routine("stepmaker", "run_stepmaker"), "boot": test_routine("kvm_tests", "run_boot"), "shutdown": test_routine("kvm_tests", "run_shutdown"), "migration": test_routine("kvm_tests", "run_migration"), "yum_update": test_routine("kvm_tests", "run_yum_update"), "autotest": test_routine("kvm_tests", "run_autotest"), - "kvm_install": test_routine("kvm_install", "run_kvm_install"), "linux_s3": test_routine("kvm_tests", "run_linux_s3"), "stress_boot": test_routine("kvm_tests", "run_stress_boot"), "timedrift": test_routine("kvm_tests", "run_timedrift"), diff --git a/client/tests/kvm/kvm_guest_wizard.py b/client/tests/kvm/kvm_guest_wizard.py deleted file mode 100644 index 8bc85f2..0000000 --- a/client/tests/kvm/kvm_guest_wizard.py +++ /dev/null @@ -1,249 +0,0 @@ -""" -Utilities to perform automatic guest installation using step files. - -@copyright: Red Hat 2008-2009 -""" - -import os, time, md5, re, shutil, logging -from autotest_lib.client.common_lib import utils, error -import kvm_utils, ppm_utils, kvm_subprocess -try: - import PIL.Image -except ImportError: - logging.warning('No python imaging library installed. PPM image ' - 'conversion to JPEG disabled. In order to enable it, ' - 'please install python-imaging or the equivalent for your ' - 'distro.') - - -def handle_var(vm, params, varname): - var = params.get(varname) - if not var: - return False - vm.send_string(var) - return True - - -def barrier_2(vm, words, params, debug_dir, data_scrdump_filename, - current_step_num): - if len(words) < 7: - logging.error("Bad barrier_2 command line") - return False - - cmd, dx, dy, x1, y1, md5sum, timeout = words[:7] - dx, dy, x1, y1, timeout = map(int, [dx, dy, x1, y1, timeout]) - - # Timeout/5 is the time it took stepmaker to complete this step. - # Divide that number by 10 to poll 10 times, just in case - # current machine is stronger then the "stepmaker machine". - # Limit to 1 (min) and 10 (max) seconds between polls. - sleep_duration = float(timeout) / 50.0 - if sleep_duration < 1.0: sleep_duration = 1.0 - if sleep_duration > 10.0: sleep_duration = 10.0 - - scrdump_filename = os.path.join(debug_dir, "scrdump.ppm") - cropped_scrdump_filename = os.path.join(debug_dir, "cropped_scrdump.ppm") - expected_scrdump_filename = os.path.join(debug_dir, "scrdump_expected.ppm") - expected_cropped_scrdump_filename = os.path.join(debug_dir, - "cropped_scrdump_expected.ppm") - comparison_filename = os.path.join(debug_dir, "comparison.ppm") - - fail_if_stuck_for = params.get("fail_if_stuck_for") - if fail_if_stuck_for: - fail_if_stuck_for = float(fail_if_stuck_for) - else: - fail_if_stuck_for = 1e308 - - stuck_detection_history = params.get("stuck_detection_history") - if stuck_detection_history: - stuck_detection_history = int(stuck_detection_history) - else: - stuck_detection_history = 2 - - keep_screendump_history = params.get("keep_screendump_history") == "yes" - if keep_screendump_history: - keep_all_history = params.get("keep_all_history") == "yes" - history_dir = os.path.join(debug_dir, "barrier_history") - - end_time = time.time() + timeout - end_time_stuck = time.time() + fail_if_stuck_for - start_time = time.time() - - prev_whole_image_md5sums = [] - - failure_message = None - - # Main loop - while True: - # Check for timeouts - if time.time() > end_time: - failure_message = "regular timeout" - break - if time.time() > end_time_stuck: - failure_message = "guest is stuck" - break - - # Make sure vm is alive - if not vm.is_alive(): - failure_message = "VM is dead" - break - - # Request screendump - (status, output) = vm.send_monitor_cmd("screendump %s" % - scrdump_filename) - if status: - logging.error("Could not fetch screendump") - continue - - # Read image file - (w, h, data) = ppm_utils.image_read_from_ppm_file(scrdump_filename) - - # Make sure image is valid - if not ppm_utils.image_verify_ppm_file(scrdump_filename): - logging.warn("Got invalid screendump: dimensions: %dx%d, " - "data size: %d" % (w, h, len(data))) - continue - - # Compute md5sum of whole image - whole_image_md5sum = ppm_utils.image_md5sum(w, h, data) - - # Write screendump to history_dir (as JPG) if requested - # and if the screendump differs from the previous one - if (keep_screendump_history and - whole_image_md5sum not in prev_whole_image_md5sums[:1]): - try: - os.makedirs(history_dir) - except: - pass - history_scrdump_filename = os.path.join(history_dir, - "scrdump-step_%s-%s.jpg" % (current_step_num, - time.strftime("%Y%m%d-%H%M%S"))) - try: - image = PIL.Image.open(scrdump_filename) - image.save(history_scrdump_filename, format = 'JPEG', - quality = 30) - except NameError: - pass - - # Compare md5sum of barrier region with the expected md5sum - calced_md5sum = ppm_utils.get_region_md5sum(w, h, data, x1, y1, dx, dy, - cropped_scrdump_filename) - if calced_md5sum == md5sum: - # Success -- remove screendump history unless requested not to - if keep_screendump_history and not keep_all_history: - shutil.rmtree(history_dir) - # Report success - return True - - # Insert image md5sum into queue of last seen images: - # If md5sum is already in queue... - if whole_image_md5sum in prev_whole_image_md5sums: - # Remove md5sum from queue - prev_whole_image_md5sums.remove(whole_image_md5sum) - else: - # Otherwise extend 'stuck' timeout - end_time_stuck = time.time() + fail_if_stuck_for - # Insert md5sum at beginning of queue - prev_whole_image_md5sums.insert(0, whole_image_md5sum) - # Limit queue length to stuck_detection_history - prev_whole_image_md5sums = \ - prev_whole_image_md5sums[:stuck_detection_history] - - # Sleep for a while - time.sleep(sleep_duration) - - # Failure - message = ("Barrier failed at step %s after %.2f seconds (%s)" % - (current_step_num, time.time() - start_time, failure_message)) - - # What should we do with this failure? - if words[-1] == "optional": - logging.info(message) - return False - else: - # Collect information and put it in debug_dir - if data_scrdump_filename and os.path.exists(data_scrdump_filename): - # Read expected screendump image - (ew, eh, edata) = \ - ppm_utils.image_read_from_ppm_file(data_scrdump_filename) - # Write it in debug_dir - ppm_utils.image_write_to_ppm_file(expected_scrdump_filename, - ew, eh, edata) - # Write the cropped version as well - ppm_utils.get_region_md5sum(ew, eh, edata, x1, y1, dx, dy, - expected_cropped_scrdump_filename) - # Perform comparison - (w, h, data) = ppm_utils.image_read_from_ppm_file(scrdump_filename) - if w == ew and h == eh: - (w, h, data) = ppm_utils.image_comparison(w, h, data, edata) - ppm_utils.image_write_to_ppm_file(comparison_filename, w, h, - data) - # Print error messages and fail the test - long_message = message + "\n(see analysis at %s)" % debug_dir - logging.error(long_message) - raise error.TestFail, message - - -def run_steps(test, params, env): - vm = kvm_utils.env_get_vm(env, params.get("main_vm")) - if not vm: - raise error.TestError("VM object not found in environment") - if not vm.is_alive(): - e_msg = "VM seems to be dead. Guestwizard requires a living VM" - raise error.TestError(e_msg) - - steps_filename = params.get("steps") - if not steps_filename: - raise error.TestError("Steps filename not specified") - steps_filename = kvm_utils.get_path(test.bindir, steps_filename) - if not os.path.exists(steps_filename): - raise error.TestError("Steps file not found: %s" % steps_filename) - - sf = open(steps_filename, "r") - lines = sf.readlines() - sf.close() - - vm.send_monitor_cmd("cont") - - current_step_num = 0 - current_screendump = None - skip_current_step = False - - # Iterate over the lines in the file - for line in lines: - line = line.strip() - if not line: - continue - logging.info(line) - - if line.startswith("#"): - continue - - words = line.split() - if words[0] == "step": - current_step_num += 1 - current_screendump = None - skip_current_step = False - elif words[0] == "screendump": - current_screendump = words[1] - elif skip_current_step: - continue - elif words[0] == "sleep": - time.sleep(float(words[1])) - elif words[0] == "key": - vm.send_key(words[1]) - elif words[0] == "var": - if not handle_var(vm, params, words[1]): - logging.error("Variable not defined: %s" % words[1]) - elif words[0] == "barrier_2": - if current_screendump: - scrdump_filename = os.path.join( - ppm_utils.get_data_dir(steps_filename), - current_screendump) - else: - scrdump_filename = None - if not barrier_2(vm, words, params, test.debugdir, - scrdump_filename, current_step_num): - skip_current_step = True - else: - vm.send_key(words[0]) diff --git a/client/tests/kvm/kvm_install.py b/client/tests/kvm/kvm_install.py deleted file mode 100755 index 148513f..0000000 --- a/client/tests/kvm/kvm_install.py +++ /dev/null @@ -1,333 +0,0 @@ -import time, os, sys, urllib, re, signal, logging, datetime -from autotest_lib.client.bin import utils, test -from autotest_lib.client.common_lib import error -import kvm_utils - - -def load_kvm_modules(module_dir): - """ - Unload previously loaded kvm modules, then load modules present on any - sub directory of module_dir. Function will walk through module_dir until - it finds the modules. - - @param module_dir: Directory where the KVM modules are located. - """ - vendor = "intel" - if os.system("grep vmx /proc/cpuinfo 1>/dev/null") != 0: - vendor = "amd" - logging.debug("Detected CPU vendor as '%s'" %(vendor)) - - logging.debug("Killing any qemu processes that might be left behind") - utils.system("pkill qemu", ignore_status=True) - - logging.info("Unloading previously loaded KVM modules") - kvm_utils.unload_module("kvm") - if utils.module_is_loaded("kvm"): - message = "Failed to remove old KVM modules" - logging.error(message) - raise error.TestError(message) - - logging.info("Loading new KVM modules...") - kvm_module_path = None - kvm_vendor_module_path = None - # Search for the built KVM modules - for folder, subdirs, files in os.walk(module_dir): - if "kvm.ko" in files: - kvm_module_path = os.path.join(folder, "kvm.ko") - kvm_vendor_module_path = os.path.join(folder, "kvm-%s.ko" % vendor) - abort = False - if not kvm_module_path: - logging.error("Need a directory containing both kernel module and " - "userspace sources.") - logging.error("If you are trying to build only KVM userspace and use " - "the KVM modules you have already loaded, put " - "'load_modules': 'no' on the control file 'params' " - "dictionary.") - raise error.TestError("Could not find a built kvm.ko module on the " - "source dir.") - elif not os.path.isfile(kvm_vendor_module_path): - logging.error("Could not find KVM (%s) module that was supposed to be" - " built on the source dir", vendor) - abort = True - if abort: - raise error.TestError("Could not load KVM modules.") - utils.system("/sbin/insmod %s" % kvm_module_path) - time.sleep(1) - utils.system("/sbin/insmod %s" % kvm_vendor_module_path) - - if not utils.module_is_loaded("kvm"): - message = "Failed to load the KVM modules built for the test" - logging.error(message) - raise error.TestError(message) - - -def create_symlinks(test_bindir, prefix): - """ - Create symbolic links for the appropriate qemu and qemu-img commands on - the kvm test bindir. - - @param test_bindir: KVM test bindir - @param prefix: KVM prefix path - """ - qemu_path = os.path.join(test_bindir, "qemu") - qemu_img_path = os.path.join(test_bindir, "qemu-img") - if os.path.lexists(qemu_path): - os.unlink(qemu_path) - if os.path.lexists(qemu_img_path): - os.unlink(qemu_img_path) - kvm_qemu = os.path.join(prefix, "bin", "qemu-system-x86_64") - if not os.path.isfile(kvm_qemu): - raise error.TestError('Invalid qemu path') - kvm_qemu_img = os.path.join(prefix, "bin", "qemu-img") - if not os.path.isfile(kvm_qemu_img): - raise error.TestError('Invalid qemu-img path') - os.symlink(kvm_qemu, qemu_path) - os.symlink(kvm_qemu_img, qemu_img_path) - - -class SourceDirInstaller: - """ - Class that handles building/installing KVM directly from a tarball or - a single source code dir. - """ - def __init__(self, test, params): - """ - Initializes class attributes, and retrieves KVM code. - - @param test: kvm test object - @param params: Dictionary with test arguments - """ - install_mode = params["mode"] - srcdir = params.get("srcdir") - # KVM build prefix - self.test_bindir = test.bindir - prefix = os.path.join(test.bindir, 'build') - self.prefix = os.path.abspath(prefix) - # Are we going to load modules? - load_modules = params.get('load_modules') - if not load_modules: - self.load_modules = True - elif load_modules == 'yes': - self.load_modules = True - elif load_modules == 'no': - self.load_modules = False - - if install_mode == 'localsrc': - if not srcdir: - raise error.TestError("Install from source directory specified" - "but no source directory provided on the" - "control file.") - else: - self.srcdir = srcdir - self.repo_type = kvm_utils.check_kvm_source_dir(self.srcdir) - return - else: - srcdir = test.srcdir - if not os.path.isdir(srcdir): - os.makedirs(srcdir) - - if install_mode == 'release': - release_tag = params.get("release_tag") - release_dir = params.get("release_dir") - logging.info("Installing KVM from release tarball") - if not release_tag: - release_tag = kvm_utils.get_latest_kvm_release_tag(release_dir) - tarball = os.path.join(release_dir, "kvm-%s.tar.gz" % release_tag) - logging.info("Retrieving release kvm-%s" % release_tag) - tarball = utils.unmap_url("/", tarball, "/tmp") - - elif install_mode == 'snapshot': - logging.info("Installing KVM from snapshot") - snapshot_dir = params.get("snapshot_dir") - if not snapshot_dir: - raise error.TestError("Snapshot dir not provided") - snapshot_date = params.get("snapshot_date") - if not snapshot_date: - # Take yesterday's snapshot - d = (datetime.date.today() - - datetime.timedelta(1)).strftime("%Y%m%d") - else: - d = snapshot_date - tarball = os.path.join(snapshot_dir, "kvm-snapshot-%s.tar.gz" % d) - logging.info("Retrieving kvm-snapshot-%s" % d) - tarball = utils.unmap_url("/", tarball, "/tmp") - - elif install_mode == 'localtar': - tarball = params.get("tarball") - if not tarball: - raise error.TestError("KVM Tarball install specified but no" - " tarball provided on control file.") - logging.info("Installing KVM from a local tarball") - logging.info("Using tarball %s") - tarball = utils.unmap_url("/", params.get("tarball"), "/tmp") - - os.chdir(srcdir) - self.srcdir = os.path.join(srcdir, utils.extract_tarball(tarball)) - self.repo_type = kvm_utils.check_kvm_source_dir(self.srcdir) - - - def __build(self): - os.chdir(self.srcdir) - # For testing purposes, it's better to build qemu binaries with - # debugging symbols, so we can extract more meaningful stack traces. - cfg = "./configure --disable-strip --prefix=%s" % self.prefix - steps = [cfg, "make clean", "make -j %s" % (utils.count_cpus() + 1)] - logging.info("Building KVM") - for step in steps: - utils.system(step) - - - def __install(self): - os.chdir(self.srcdir) - logging.info("Installing KVM userspace") - if self.repo_type == 1: - utils.system("make -C qemu install") - elif self.repo_type == 2: - utils.system("make install") - create_symlinks(self.test_bindir, self.prefix) - - - def __load_modules(self): - load_kvm_modules(self.srcdir) - - - def install(self): - self.__build() - self.__install() - if self.load_modules: - self.__load_modules() - - -class GitInstaller: - def __init__(self, test, params): - """ - Initialize class parameters and retrieves code from git repositories. - - @param test: kvm test object. - @param params: Dictionary with test parameters. - """ - install_mode = params["mode"] - srcdir = params.get("srcdir", test.bindir) - if not srcdir: - os.makedirs(srcdir) - self.srcdir = srcdir - # KVM build prefix - self.test_bindir = test.bindir - prefix = os.path.join(test.bindir, 'build') - self.prefix = os.path.abspath(prefix) - # Are we going to load modules? - load_modules = params.get('load_modules') - if not load_modules: - self.load_modules = True - elif load_modules == 'yes': - self.load_modules = True - elif load_modules == 'no': - self.load_modules = False - - kernel_repo = params.get("git_repo") - user_repo = params.get("user_git_repo") - kmod_repo = params.get("kmod_repo") - - branch = params.get("git_branch", "master") - lbranch = params.get("lbranch") - - tag = params.get("git_tag", "HEAD") - user_tag = params.get("user_git_tag", "HEAD") - kmod_tag = params.get("kmod_git_tag", "HEAD") - - if not kernel_repo: - message = "KVM git repository path not specified" - logging.error(message) - raise error.TestError(message) - if not user_repo: - message = "KVM user git repository path not specified" - logging.error(message) - raise error.TestError(message) - - kernel_srcdir = os.path.join(srcdir, "kvm") - kvm_utils.get_git_branch(kernel_repo, branch, kernel_srcdir, tag, - lbranch) - self.kernel_srcdir = kernel_srcdir - - userspace_srcdir = os.path.join(srcdir, "kvm_userspace") - kvm_utils.get_git_branch(user_repo, branch, userspace_srcdir, user_tag, - lbranch) - self.userspace_srcdir = userspace_srcdir - - if kmod_repo: - kmod_srcdir = os.path.join (srcdir, "kvm_kmod") - kvm_utils.get_git_branch(kmod_repo, branch, kmod_srcdir, user_tag, - lbranch) - self.kmod_srcdir = kmod_srcdir - - - def __build(self): - if self.kmod_srcdir: - logging.info('Building KVM modules') - os.chdir(self.kmod_srcdir) - utils.system('./configure') - utils.system('make clean') - utils.system('make sync LINUX=%s' % self.kernel_srcdir) - utils.system('make -j %s' % utils.count_cpus()) - logging.info('Building KVM userspace code') - os.chdir(self.userspace_srcdir) - utils.system('./configure --disable-strip --prefix=%s' % - self.prefix) - utils.system('make clean') - utils.system('make -j %s' % utils.count_cpus()) - else: - os.chdir(self.userspace_srcdir) - utils.system('./configure --disable-strip --prefix=%s' % - self.prefix) - logging.info('Building KVM modules') - utils.system('make clean') - utils.system('make -C kernel LINUX=%s sync' % self.kernel_srcdir) - logging.info('Building KVM userspace code') - utils.system('make -j %s' % utils.count_cpus()) - - - def __install(self): - os.chdir(self.userspace_srcdir) - utils.system('make install') - create_symlinks(self.test_bindir, self.prefix) - - - def __load_modules(self): - if self.kmod_srcdir: - load_kvm_modules(self.kmod_srcdir) - else: - load_kvm_modules(self.userspace_srcdir) - - - def install(self): - self.__build() - self.__install() - if self.load_modules: - self.__load_modules() - - -def run_kvm_install(test, params, env): - """ - Installs KVM using the selected install mode. Most install methods will - take kvm source code, build it and install it to a given location. - - @param test: kvm test object. - @param params: Dictionary with test parameters. - @param env: Test environment. - """ - install_mode = params.get("mode") - srcdir = params.get("srcdir", test.srcdir) - params["srcdir"] = srcdir - - if install_mode == 'noinstall': - logging.info("Skipping installation") - return - elif install_mode in ['localsrc', 'localtar', 'release', 'snapshot']: - installer = SourceDirInstaller(test, params) - elif install_mode == 'git': - installer = GitInstaller(test, params) - else: - raise error.TestError('Invalid or unsupported' - ' install mode: %s' % install_mode) - - installer.install() diff --git a/client/tests/kvm/steps.py b/client/tests/kvm/steps.py new file mode 100644 index 0000000..8bc85f2 --- /dev/null +++ b/client/tests/kvm/steps.py @@ -0,0 +1,249 @@ +""" +Utilities to perform automatic guest installation using step files. + +@copyright: Red Hat 2008-2009 +""" + +import os, time, md5, re, shutil, logging +from autotest_lib.client.common_lib import utils, error +import kvm_utils, ppm_utils, kvm_subprocess +try: + import PIL.Image +except ImportError: + logging.warning('No python imaging library installed. PPM image ' + 'conversion to JPEG disabled. In order to enable it, ' + 'please install python-imaging or the equivalent for your ' + 'distro.') + + +def handle_var(vm, params, varname): + var = params.get(varname) + if not var: + return False + vm.send_string(var) + return True + + +def barrier_2(vm, words, params, debug_dir, data_scrdump_filename, + current_step_num): + if len(words) < 7: + logging.error("Bad barrier_2 command line") + return False + + cmd, dx, dy, x1, y1, md5sum, timeout = words[:7] + dx, dy, x1, y1, timeout = map(int, [dx, dy, x1, y1, timeout]) + + # Timeout/5 is the time it took stepmaker to complete this step. + # Divide that number by 10 to poll 10 times, just in case + # current machine is stronger then the "stepmaker machine". + # Limit to 1 (min) and 10 (max) seconds between polls. + sleep_duration = float(timeout) / 50.0 + if sleep_duration < 1.0: sleep_duration = 1.0 + if sleep_duration > 10.0: sleep_duration = 10.0 + + scrdump_filename = os.path.join(debug_dir, "scrdump.ppm") + cropped_scrdump_filename = os.path.join(debug_dir, "cropped_scrdump.ppm") + expected_scrdump_filename = os.path.join(debug_dir, "scrdump_expected.ppm") + expected_cropped_scrdump_filename = os.path.join(debug_dir, + "cropped_scrdump_expected.ppm") + comparison_filename = os.path.join(debug_dir, "comparison.ppm") + + fail_if_stuck_for = params.get("fail_if_stuck_for") + if fail_if_stuck_for: + fail_if_stuck_for = float(fail_if_stuck_for) + else: + fail_if_stuck_for = 1e308 + + stuck_detection_history = params.get("stuck_detection_history") + if stuck_detection_history: + stuck_detection_history = int(stuck_detection_history) + else: + stuck_detection_history = 2 + + keep_screendump_history = params.get("keep_screendump_history") == "yes" + if keep_screendump_history: + keep_all_history = params.get("keep_all_history") == "yes" + history_dir = os.path.join(debug_dir, "barrier_history") + + end_time = time.time() + timeout + end_time_stuck = time.time() + fail_if_stuck_for + start_time = time.time() + + prev_whole_image_md5sums = [] + + failure_message = None + + # Main loop + while True: + # Check for timeouts + if time.time() > end_time: + failure_message = "regular timeout" + break + if time.time() > end_time_stuck: + failure_message = "guest is stuck" + break + + # Make sure vm is alive + if not vm.is_alive(): + failure_message = "VM is dead" + break + + # Request screendump + (status, output) = vm.send_monitor_cmd("screendump %s" % + scrdump_filename) + if status: + logging.error("Could not fetch screendump") + continue + + # Read image file + (w, h, data) = ppm_utils.image_read_from_ppm_file(scrdump_filename) + + # Make sure image is valid + if not ppm_utils.image_verify_ppm_file(scrdump_filename): + logging.warn("Got invalid screendump: dimensions: %dx%d, " + "data size: %d" % (w, h, len(data))) + continue + + # Compute md5sum of whole image + whole_image_md5sum = ppm_utils.image_md5sum(w, h, data) + + # Write screendump to history_dir (as JPG) if requested + # and if the screendump differs from the previous one + if (keep_screendump_history and + whole_image_md5sum not in prev_whole_image_md5sums[:1]): + try: + os.makedirs(history_dir) + except: + pass + history_scrdump_filename = os.path.join(history_dir, + "scrdump-step_%s-%s.jpg" % (current_step_num, + time.strftime("%Y%m%d-%H%M%S"))) + try: + image = PIL.Image.open(scrdump_filename) + image.save(history_scrdump_filename, format = 'JPEG', + quality = 30) + except NameError: + pass + + # Compare md5sum of barrier region with the expected md5sum + calced_md5sum = ppm_utils.get_region_md5sum(w, h, data, x1, y1, dx, dy, + cropped_scrdump_filename) + if calced_md5sum == md5sum: + # Success -- remove screendump history unless requested not to + if keep_screendump_history and not keep_all_history: + shutil.rmtree(history_dir) + # Report success + return True + + # Insert image md5sum into queue of last seen images: + # If md5sum is already in queue... + if whole_image_md5sum in prev_whole_image_md5sums: + # Remove md5sum from queue + prev_whole_image_md5sums.remove(whole_image_md5sum) + else: + # Otherwise extend 'stuck' timeout + end_time_stuck = time.time() + fail_if_stuck_for + # Insert md5sum at beginning of queue + prev_whole_image_md5sums.insert(0, whole_image_md5sum) + # Limit queue length to stuck_detection_history + prev_whole_image_md5sums = \ + prev_whole_image_md5sums[:stuck_detection_history] + + # Sleep for a while + time.sleep(sleep_duration) + + # Failure + message = ("Barrier failed at step %s after %.2f seconds (%s)" % + (current_step_num, time.time() - start_time, failure_message)) + + # What should we do with this failure? + if words[-1] == "optional": + logging.info(message) + return False + else: + # Collect information and put it in debug_dir + if data_scrdump_filename and os.path.exists(data_scrdump_filename): + # Read expected screendump image + (ew, eh, edata) = \ + ppm_utils.image_read_from_ppm_file(data_scrdump_filename) + # Write it in debug_dir + ppm_utils.image_write_to_ppm_file(expected_scrdump_filename, + ew, eh, edata) + # Write the cropped version as well + ppm_utils.get_region_md5sum(ew, eh, edata, x1, y1, dx, dy, + expected_cropped_scrdump_filename) + # Perform comparison + (w, h, data) = ppm_utils.image_read_from_ppm_file(scrdump_filename) + if w == ew and h == eh: + (w, h, data) = ppm_utils.image_comparison(w, h, data, edata) + ppm_utils.image_write_to_ppm_file(comparison_filename, w, h, + data) + # Print error messages and fail the test + long_message = message + "\n(see analysis at %s)" % debug_dir + logging.error(long_message) + raise error.TestFail, message + + +def run_steps(test, params, env): + vm = kvm_utils.env_get_vm(env, params.get("main_vm")) + if not vm: + raise error.TestError("VM object not found in environment") + if not vm.is_alive(): + e_msg = "VM seems to be dead. Guestwizard requires a living VM" + raise error.TestError(e_msg) + + steps_filename = params.get("steps") + if not steps_filename: + raise error.TestError("Steps filename not specified") + steps_filename = kvm_utils.get_path(test.bindir, steps_filename) + if not os.path.exists(steps_filename): + raise error.TestError("Steps file not found: %s" % steps_filename) + + sf = open(steps_filename, "r") + lines = sf.readlines() + sf.close() + + vm.send_monitor_cmd("cont") + + current_step_num = 0 + current_screendump = None + skip_current_step = False + + # Iterate over the lines in the file + for line in lines: + line = line.strip() + if not line: + continue + logging.info(line) + + if line.startswith("#"): + continue + + words = line.split() + if words[0] == "step": + current_step_num += 1 + current_screendump = None + skip_current_step = False + elif words[0] == "screendump": + current_screendump = words[1] + elif skip_current_step: + continue + elif words[0] == "sleep": + time.sleep(float(words[1])) + elif words[0] == "key": + vm.send_key(words[1]) + elif words[0] == "var": + if not handle_var(vm, params, words[1]): + logging.error("Variable not defined: %s" % words[1]) + elif words[0] == "barrier_2": + if current_screendump: + scrdump_filename = os.path.join( + ppm_utils.get_data_dir(steps_filename), + current_screendump) + else: + scrdump_filename = None + if not barrier_2(vm, words, params, test.debugdir, + scrdump_filename, current_step_num): + skip_current_step = True + else: + vm.send_key(words[0])