From patchwork Thu Feb 25 22:11:41 2010 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Goldish X-Patchwork-Id: 82210 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter.kernel.org (8.14.3/8.14.3) with ESMTP id o1PMDI5x030196 for ; Thu, 25 Feb 2010 22:13:19 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S934234Ab0BYWNP (ORCPT ); Thu, 25 Feb 2010 17:13:15 -0500 Received: from mx1.redhat.com ([209.132.183.28]:4181 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S934185Ab0BYWNH (ORCPT ); Thu, 25 Feb 2010 17:13:07 -0500 Received: from int-mx01.intmail.prod.int.phx2.redhat.com (int-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.11]) by mx1.redhat.com (8.13.8/8.13.8) with ESMTP id o1PMD5dT006661 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=OK); Thu, 25 Feb 2010 17:13:05 -0500 Received: from ns3.rdu.redhat.com (ns3.rdu.redhat.com [10.11.255.199]) by int-mx01.intmail.prod.int.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id o1PMD4GV011537; Thu, 25 Feb 2010 17:13:04 -0500 Received: from localhost.localdomain (dhcp-1-188.tlv.redhat.com [10.35.1.188]) by ns3.rdu.redhat.com (8.13.8/8.13.8) with ESMTP id o1PMD1a6024131; Thu, 25 Feb 2010 17:13:02 -0500 From: Michael Goldish To: autotest@test.kernel.org, kvm@vger.kernel.org Cc: Michael Goldish Subject: [KVM-AUTOTEST PATCH v2] [RFC] KVM test: A memory efficient kvm_config implementation Date: Fri, 26 Feb 2010 00:11:41 +0200 Message-Id: <1267135901-30989-1-git-send-email-mgoldish@redhat.com> X-Scanned-By: MIMEDefang 2.67 on 10.5.11.11 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.3 (demeter.kernel.org [140.211.167.41]); Thu, 25 Feb 2010 22:13:20 +0000 (UTC) diff --git a/client/tests/kvm/control b/client/tests/kvm/control index 163286e..a3bba94 100644 --- a/client/tests/kvm/control +++ b/client/tests/kvm/control @@ -30,34 +30,38 @@ import kvm_utils, kvm_config # set English environment (command output might be localized, need to be safe) os.environ['LANG'] = 'en_US.UTF-8' -build_cfg_path = os.path.join(kvm_test_dir, "build.cfg") -build_cfg = kvm_config.config(build_cfg_path) -# Make any desired changes to the build configuration here. For example: -#build_cfg.parse_string(""" +str = """ +# This string will be parsed after build.cfg. Make any desired changes to the +# build configuration here. For example: #release_tag = 84 -#""") -if not kvm_utils.run_tests(build_cfg.get_list(), job): +""" +build_cfg = kvm_config.config() +build_cfg_path = os.path.join(kvm_test_dir, "build.cfg") +build_cfg.fork_and_parse(build_cfg_path, str) +if not kvm_utils.run_tests(build_cfg.get_generator(), job): logging.error("KVM build step failed, exiting.") sys.exit(1) -tests_cfg_path = os.path.join(kvm_test_dir, "tests.cfg") -tests_cfg = kvm_config.config(tests_cfg_path) -# Make any desired changes to the test configuration here. For example: -#tests_cfg.parse_string(""" +str = """ +# This string will be parsed after tests.cfg. Make any desired configuration +# changes here. For example: #display = sdl #install|setup: timeout_multiplier = 3 -#""") +""" +tests_cfg = kvm_config.config() +tests_cfg_path = os.path.join(kvm_test_dir, "tests.cfg") +tests_cfg.fork_and_parse(tests_cfg_path, str) pools_cfg_path = os.path.join(kvm_test_dir, "address_pools.cfg") tests_cfg.parse_file(pools_cfg_path) hostname = os.uname()[1].split(".")[0] -if tests_cfg.filter("^" + hostname): +if tests_cfg.count("^" + hostname): tests_cfg.parse_string("only ^%s" % hostname) else: tests_cfg.parse_string("only ^default_host") # Run the tests -kvm_utils.run_tests(tests_cfg.get_list(), job) +kvm_utils.run_tests(tests_cfg.get_generator(), job) # Generate a nice HTML report inside the job's results dir kvm_utils.create_report(kvm_test_dir, job.resultdir) diff --git a/client/tests/kvm/control.parallel b/client/tests/kvm/control.parallel index 343f694..07bc6e5 100644 --- a/client/tests/kvm/control.parallel +++ b/client/tests/kvm/control.parallel @@ -160,19 +160,22 @@ if not params.get("mode") == "noinstall": # ---------------------------------------------------------- import kvm_config -filename = os.path.join(pwd, "kvm_tests.cfg") -cfg = kvm_config.config(filename) - -# If desirable, make changes to the test configuration here. For example: -# cfg.parse_string("install|setup: timeout_multiplier = 2") -# cfg.parse_string("only fc8_quick") -# cfg.parse_string("display = sdl") +str = """ +# This string will be parsed after tests.cfg. Make any desired changes to the +# test configuration here. For example: +#install|setup: timeout_multiplier = 3 +#only fc8_quick +#display = sdl +""" +cfg = kvm_config.config() +filename = os.path.join(pwd, "tests.cfg") +cfg.fork_and_parse(filename, str) -filename = os.path.join(pwd, "kvm_address_pools.cfg") +filename = os.path.join(pwd, "address_pools.cfg") if os.path.exists(filename): cfg.parse_file(filename) hostname = os.uname()[1].split(".")[0] - if cfg.filter("^" + hostname): + if cfg.count("^" + hostname): cfg.parse_string("only ^%s" % hostname) else: cfg.parse_string("only ^default_host") diff --git a/client/tests/kvm/kvm_config.py b/client/tests/kvm/kvm_config.py index 798ef56..72d7ac6 100755 --- a/client/tests/kvm/kvm_config.py +++ b/client/tests/kvm/kvm_config.py @@ -2,10 +2,10 @@ """ KVM configuration file utility functions. -@copyright: Red Hat 2008-2009 +@copyright: Red Hat 2008-2010 """ -import logging, re, os, sys, StringIO, optparse +import logging, re, os, sys, StringIO, optparse, array, cPickle, traceback import common from autotest_lib.client.common_lib import error from autotest_lib.client.common_lib import logging_config, logging_manager @@ -21,29 +21,29 @@ class config: """ Parse an input file or string that follows the KVM Test Config File format and generate a list of dicts that will be later used as configuration - parameters by the the KVM tests. + parameters by the KVM tests. @see: http://www.linux-kvm.org/page/KVM-Autotest/Test_Config_File """ - def __init__(self, filename=None, debug=False): + def __init__(self, filename=None, debug=True): """ - Initialize the list and optionally parse filename. + Initialize the list and optionally parse a file. @param filename: Path of the file that will be taken. @param debug: Whether to turn debugging output. """ - self.list = [{"name": "", "shortname": "", "depend": []}] - self.debug = debug + self.list = [array.array("H", [4, 4, 4, 4])] + self.object_cache = [] self.filename = filename + self.debug = debug if filename: self.parse_file(filename) def parse_file(self, filename): """ - Parse filename, return the resulting list and store it in .list. If - filename does not exist, raise an exception. + Parse file. If it doesn't exist, raise an exception. @param filename: Path of the configuration file. """ @@ -53,83 +53,121 @@ class config: file = open(filename, "r") self.list = self.parse(file, self.list) file.close() - return self.list def parse_string(self, str): """ - Parse a string, return the resulting list and store it in .list. + Parse a string. - @param str: String that will be parsed. + @param str: String to parse. """ file = StringIO.StringIO(str) self.list = self.parse(file, self.list) file.close() - return self.list - def get_list(self): + def fork_and_parse(self, filename=None, str=None): """ - Return the list of dictionaries. This should probably be called after - parsing something. - """ - return self.list + Parse a file and/or a string in a separate process to save memory. + + Python likes to keep memory to itself even after the objects occupying + it have been destroyed. If during a call to parse_file() or + parse_string() a lot of memory is used, it can only be freed by + terminating the process. This function works around the problem by + doing the parsing in a forked process and then terminating it, freeing + any unneeded memory. + Note: if an exception is raised during parsing, its information will be + printed, and the resulting list will be empty. The exception will not + be raised in the process calling this function. - def match(self, filter, dict): + @param filename: Path of file to parse (optional). + @param str: String to parse (optional). + """ + pipe_r, pipe_w = os.pipe() + pipe_r = os.fdopen(pipe_r, "r") + pipe_w = os.fdopen(pipe_w, "w") + pid = os.fork() + if not pid: + # Child process + try: + if filename: + self.parse_file(filename) + if str: + self.parse_string(str) + except: + traceback.print_exc() + self.list = [] + l = [a.tostring() for a in self.list] + cPickle.dump((l, self.object_cache), pipe_w, -1) + pipe_w.close() + os._exit(0) + else: + # Parent process + (l, self.object_cache) = cPickle.load(pipe_r) + pipe_r.close() + os.waitpid(pid, 0) + self.list = [] + for s in l: + a = array.array("H") + a.fromstring(s) + self.list.append(a) + + + def get_generator(self): """ - Return True if dict matches filter. + Generate dictionaries from the code parsed so far. This should + probably be called after parsing something. - @param filter: A regular expression that defines the filter. - @param dict: Dictionary that will be inspected. + @return: A dict generator. """ - filter = re.compile(r"(\.|^)(%s)(\.|$)" % filter) - return bool(filter.search(dict["name"])) + for a in self.list: + name, shortname, depend, content = _array_get_all(a, self.object_cache) + dict = {"name": name, "shortname": shortname, "depend": depend} + _apply_content_to_dict(dict, content) + yield dict - def filter(self, filter, list=None): + def get_list(self): """ - Filter a list of dicts. + Generate a list of dictionaries from the code parsed so far. + This should probably be called after parsing something. - @param filter: A regular expression that will be used as a filter. - @param list: A list of dictionaries that will be filtered. + @return: A list of dicts. """ - if list is None: - list = self.list - return [dict for dict in list if self.match(filter, dict)] + return list(self.get_generator()) - def split_and_strip(self, str, sep="="): + def count(self, filter=".*"): """ - Split str and strip quotes from the resulting parts. + Return the number of dictionaries whose names match filter. - @param str: String that will be processed - @param sep: Separator that will be used to split the string + @param filter: A regular expression string. """ - temp = str.split(sep, 1) - for i in range(len(temp)): - temp[i] = temp[i].strip() - if re.findall("^\".*\"$", temp[i]): - temp[i] = temp[i].strip("\"") - elif re.findall("^\'.*\'$", temp[i]): - temp[i] = temp[i].strip("\'") - return temp + exp = _get_filter_regex(filter) + count = 0 + for a in self.list: + name = _array_get_name(a, self.object_cache) + if exp.search(name): + count += 1 + return count def get_next_line(self, file): """ Get the next non-empty, non-comment line in a file like object. - @param file: File like object + @param file: File like object. @return: If no line is available, return None. """ while True: line = file.readline() - if line == "": return None + if not line: + return None stripped_line = line.strip() - if len(stripped_line) > 0 \ - and not stripped_line.startswith('#') \ - and not stripped_line.startswith('//'): + if (stripped_line + and not stripped_line.startswith('#') + and not stripped_line.startswith('//')): return line @@ -142,34 +180,11 @@ class config: """ pos = file.tell() line = self.get_next_line(file) + file.seek(pos) if not line: - file.seek(pos) return -1 line = line.expandtabs() - indent = 0 - while line[indent] == ' ': - indent += 1 - file.seek(pos) - return indent - - - def add_name(self, str, name, append=False): - """ - Add name to str with a separator dot and return the result. - - @param str: String that will be processed - @param name: name that will be appended to the string. - @return: If append is True, append name to str. - Otherwise, pre-pend name to str. - """ - if str == "": - return name - # Append? - elif append: - return str + "." + name - # Prepend? - else: - return name + "." + str + return len(line) - len(line.lstrip()) def parse_variants(self, file, list, subvariants=False, prev_indent=-1): @@ -178,13 +193,13 @@ class config: level lower than or equal to prev_indent is encountered. @brief: Parse a 'variants' or 'subvariants' block from a file-like - object. - @param file: File-like object that will be parsed - @param list: List of dicts to operate on + object. + @param file: File-like object that will be parsed. + @param list: List of arrays to operate on. @param subvariants: If True, parse in 'subvariants' mode; - otherwise parse in 'variants' mode - @param prev_indent: The indent level of the "parent" block - @return: The resulting list of dicts. + otherwise parse in 'variants' mode. + @param prev_indent: The indent level of the "parent" block. + @return: The resulting list of arrays. """ new_list = [] @@ -193,46 +208,47 @@ class config: if indent <= prev_indent: break indented_line = self.get_next_line(file).rstrip() - line = indented_line.strip() + line = indented_line.lstrip() # Get name and dependencies - temp = line.strip("- ").split(":") - name = temp[0] - if len(temp) == 1: - dep_list = [] - else: - dep_list = temp[1].split() + (name, depend) = line.strip("- ").split(":") # See if name should be added to the 'shortname' field - add_to_shortname = True - if name.startswith("@"): - name = name.strip("@") - add_to_shortname = False - - # Make a deep copy of list - temp_list = [] - for dict in list: - new_dict = dict.copy() - new_dict["depend"] = dict["depend"][:] - temp_list.append(new_dict) + add_to_shortname = not name.startswith("@") + name = name.lstrip("@") + + # Store name and dependencies in cache and get their indices + n = self._store_str(name) + d = self._store_str(depend) + + # Make a copy of list + temp_list = [a[:] for a in list] if subvariants: # If we're parsing 'subvariants', first modify the list - self.__modify_list_subvariants(temp_list, name, dep_list, - add_to_shortname) - temp_list = self.parse(file, temp_list, - restricted=True, prev_indent=indent) + if add_to_shortname: + for a in temp_list: + _array_append_to_name_shortname_depend(a, n, d) + else: + for a in temp_list: + _array_append_to_name_depend(a, n, d) + temp_list = self.parse(file, temp_list, restricted=False, + prev_indent=indent) else: # If we're parsing 'variants', parse before modifying the list if self.debug: - self.__debug_print(indented_line, - "Entering variant '%s' " - "(variant inherits %d dicts)" % - (name, len(list))) - temp_list = self.parse(file, temp_list, - restricted=False, prev_indent=indent) - self.__modify_list_variants(temp_list, name, dep_list, - add_to_shortname) + _debug_print(indented_line, + "Entering variant '%s' " + "(variant inherits %d dicts)" % + (name, len(list))) + temp_list = self.parse(file, temp_list, restricted=False, + prev_indent=indent) + if add_to_shortname: + for a in temp_list: + _array_prepend_to_name_shortname_depend(a, n, d) + else: + for a in temp_list: + _array_prepend_to_name_depend(a, n, d) new_list += temp_list @@ -246,123 +262,93 @@ class config: @brief: Parse a file-like object. @param file: A file-like object - @param list: A list of dicts to operate on (list is modified in - place and should not be used after the call) - @param restricted: if True, operate in restricted mode - (prohibit 'variants') - @param prev_indent: the indent level of the "parent" block - @return: Return the resulting list of dicts. + @param list: A list of arrays to operate on (list is modified in + place and should not be used after the call). + @param restricted: If True, operate in restricted mode + (prohibit 'variants'). + @param prev_indent: The indent level of the "parent" block. + @return: The resulting list of arrays. @note: List is destroyed and should not be used after the call. - Only the returned list should be used. + Only the returned list should be used. """ + current_block = "" + while True: indent = self.get_next_line_indent(file) if indent <= prev_indent: + self._append_content_to_arrays(list, current_block) break indented_line = self.get_next_line(file).rstrip() - line = indented_line.strip() - words = line.split() + line = indented_line.lstrip() len_list = len(list) - # Look for a known operator in the line - operators = ["?+=", "?<=", "?=", "+=", "<=", "="] - op_found = None - op_pos = len(line) - for op in operators: - pos = line.find(op) - if pos >= 0 and pos < op_pos: - op_found = op - op_pos = pos - - # Found an operator? - if op_found: + # Parse assignment operators (keep lines in temporary buffer) + if "=" in line: if self.debug and not restricted: - self.__debug_print(indented_line, - "Parsing operator (%d dicts in current " - "context)" % len_list) - (left, value) = self.split_and_strip(line, op_found) - filters_and_key = self.split_and_strip(left, ":") - filters = filters_and_key[:-1] - key = filters_and_key[-1] - filtered_list = list - for filter in filters: - filtered_list = self.filter(filter, filtered_list) - # Apply the operation to the filtered list - if op_found == "=": - for dict in filtered_list: - dict[key] = value - elif op_found == "+=": - for dict in filtered_list: - dict[key] = dict.get(key, "") + value - elif op_found == "<=": - for dict in filtered_list: - dict[key] = value + dict.get(key, "") - elif op_found.startswith("?"): - exp = re.compile("^(%s)$" % key) - if op_found == "?=": - for dict in filtered_list: - for key in dict.keys(): - if exp.match(key): - dict[key] = value - elif op_found == "?+=": - for dict in filtered_list: - for key in dict.keys(): - if exp.match(key): - dict[key] = dict.get(key, "") + value - elif op_found == "?<=": - for dict in filtered_list: - for key in dict.keys(): - if exp.match(key): - dict[key] = value + dict.get(key, "") + _debug_print(indented_line, + "Parsing operator (%d dicts in current " + "context)" % len_list) + current_block += line + "\n" + continue + + # Flush the temporary buffer + self._append_content_to_arrays(list, current_block) + current_block = "" + + words = line.split() # Parse 'no' and 'only' statements - elif words[0] == "no" or words[0] == "only": + if words[0] == "no" or words[0] == "only": if len(words) <= 1: continue - filters = words[1:] + filters = map(_get_filter_regex, words[1:]) filtered_list = [] if words[0] == "no": - for dict in list: + for a in list: + name = _array_get_name(a, self.object_cache) for filter in filters: - if self.match(filter, dict): + if filter.search(name): break else: - filtered_list.append(dict) + filtered_list.append(a) if words[0] == "only": - for dict in list: + for a in list: + name = _array_get_name(a, self.object_cache) for filter in filters: - if self.match(filter, dict): - filtered_list.append(dict) + if filter.search(name): + filtered_list.append(a) break list = filtered_list if self.debug and not restricted: - self.__debug_print(indented_line, - "Parsing no/only (%d dicts in current " - "context, %d remain)" % - (len_list, len(list))) + _debug_print(indented_line, + "Parsing no/only (%d dicts in current " + "context, %d remain)" % + (len_list, len(list))) + continue # Parse 'variants' - elif line == "variants:": + if line == "variants:": # 'variants' not allowed in restricted mode # (inside an exception or inside subvariants) if restricted: e_msg = "Using variants in this context is not allowed" raise error.AutotestError(e_msg) if self.debug and not restricted: - self.__debug_print(indented_line, - "Entering variants block (%d dicts in " - "current context)" % len_list) + _debug_print(indented_line, + "Entering variants block (%d dicts in " + "current context)" % len_list) list = self.parse_variants(file, list, subvariants=False, prev_indent=indent) + continue # Parse 'subvariants' (the block is parsed for each dict # separately) - elif line == "subvariants:": + if line == "subvariants:": if self.debug and not restricted: - self.__debug_print(indented_line, - "Entering subvariants block (%d dicts in " - "current context)" % len_list) + _debug_print(indented_line, + "Entering subvariants block (%d dicts in " + "current context)" % len_list) new_list = [] # Remember current file position pos = file.tell() @@ -370,7 +356,7 @@ class config: self.parse_variants(file, [], subvariants=True, prev_indent=indent) # Iterate over the list... - for index in range(len(list)): + for index in xrange(len(list)): # Revert to initial file position in this 'subvariants' # block file.seek(pos) @@ -380,14 +366,15 @@ class config: subvariants=True, prev_indent=indent) list = new_list + continue # Parse 'include' statements - elif words[0] == "include": + if words[0] == "include": if len(words) <= 1: continue if self.debug and not restricted: - self.__debug_print(indented_line, - "Entering file %s" % words[1]) + _debug_print(indented_line, + "Entering file %s" % words[1]) if self.filename: filename = os.path.join(os.path.dirname(self.filename), words[1]) @@ -396,22 +383,23 @@ class config: list = self.parse(new_file, list, restricted) new_file.close() if self.debug and not restricted: - self.__debug_print("", "Leaving file %s" % words[1]) + _debug_print("", "Leaving file %s" % words[1]) else: logging.warning("Cannot include %s -- file not found", filename) else: logging.warning("Cannot include %s because no file is " "currently open", words[1]) + continue # Parse multi-line exceptions # (the block is parsed for each dict separately) - elif line.endswith(":"): + if line.endswith(":"): if self.debug and not restricted: - self.__debug_print(indented_line, - "Entering multi-line exception block " - "(%d dicts in current context outside " - "exception)" % len_list) + _debug_print(indented_line, + "Entering multi-line exception block " + "(%d dicts in current context outside " + "exception)" % len_list) line = line.strip(":") new_list = [] # Remember current file position @@ -419,8 +407,11 @@ class config: # Read the lines in any case self.parse(file, [], restricted=True, prev_indent=indent) # Iterate over the list... - for index in range(len(list)): - if self.match(line, list[index]): + exp = _get_filter_regex(line) + for index in xrange(len(list)): + name = _array_get_name(list[index], self.object_cache) + if exp.search(name): + #if self.match(line, list[index]): # Revert to initial file position in this # exception block file.seek(pos) @@ -430,81 +421,243 @@ class config: restricted=True, prev_indent=indent) else: - new_list += list[index:index+1] + #new_list += list[index:index+1] + new_list.append(list[index]) list = new_list + continue return list - def __debug_print(self, str1, str2=""): + def _store_str(self, str): """ - Nicely print two strings and an arrow. + Store str in the internal object cache, if it isn't already there, and + return its identifying index. - @param str1: First string - @param str2: Second string + @param str: String to store. + @return: The index of str in the object cache. """ - if str2: - str = "%-50s ---> %s" % (str1, str2) - else: - str = str1 - logging.debug(str) + try: + return self.object_cache.index(str) + except ValueError: + self.object_cache.append(str) + return len(self.object_cache) - 1 - def __modify_list_variants(self, list, name, dep_list, add_to_shortname): - """ - Make some modifications to list, as part of parsing a 'variants' block. - - @param list: List to be processed - @param name: Name to be prepended to the dictionary's 'name' key - @param dep_list: List of dependencies to be added to the dictionary's - 'depend' key - @param add_to_shortname: Boolean indicating whether name should be - prepended to the dictionary's 'shortname' key as well + def _append_content_to_arrays(self, list, content): """ - for dict in list: - # Prepend name to the dict's 'name' field - dict["name"] = self.add_name(dict["name"], name) - # Prepend name to the dict's 'shortname' field - if add_to_shortname: - dict["shortname"] = self.add_name(dict["shortname"], name) - # Prepend name to each of the dict's dependencies - for i in range(len(dict["depend"])): - dict["depend"][i] = self.add_name(dict["depend"][i], name) - # Add new dependencies - dict["depend"] += dep_list - - - def __modify_list_subvariants(self, list, name, dep_list, add_to_shortname): - """ - Make some modifications to list, as part of parsing a 'subvariants' - block. - - @param list: List to be processed - @param name: Name to be appended to the dictionary's 'name' key - @param dep_list: List of dependencies to be added to the dictionary's - 'depend' key - @param add_to_shortname: Boolean indicating whether name should be - appended to the dictionary's 'shortname' as well + Append content (config code containing assignment operations) to a list + of arrays. + + @param list: List of arrays to operate on. + @param content: String containing assignment operations. """ - for dict in list: - # Add new dependencies - for dep in dep_list: - dep_name = self.add_name(dict["name"], dep, append=True) - dict["depend"].append(dep_name) - # Append name to the dict's 'name' field - dict["name"] = self.add_name(dict["name"], name, append=True) - # Append name to the dict's 'shortname' field - if add_to_shortname: - dict["shortname"] = self.add_name(dict["shortname"], name, - append=True) + if content: + str_index = self._store_str(content) + for a in list: + _array_append_to_content(a, str_index) + + + +# Assignment operators: + +def _op_set(dict, key, value): + dict[key] = value + + +def _op_append(dict, key, value): + dict[key] = dict.get(key, "") + value + + +def _op_prepend(dict, key, value): + dict[key] = value + dict.get(key, "") + + +def _op_regex_set(dict, exp, value): + exp = re.compile("^(%s)$" % exp) + for key in dict: + if exp.match(key): + dict[key] = value + + +def _op_regex_append(dict, exp, value): + exp = re.compile("^(%s)$" % exp) + for key in dict: + if exp.match(key): + dict[key] += value + + +def _op_regex_prepend(dict, exp, value): + exp = re.compile("^(%s)$" % exp) + for key in dict: + if exp.match(key): + dict[key] = value + dict[key] + + +ops = { + "=": _op_set, + "+=": _op_append, + "<=": _op_prepend, + "?=": _op_regex_set, + "?+=": _op_regex_append, + "?<=": _op_regex_prepend, +} + + +def _apply_content_to_dict(dict, content): + """ + Apply the operations in content (config code containing assignment + operations) to a dict. + + @param dict: Dictionary to operate on. Must have 'name' key. + @param content: String containing assignment operations. + """ + for line in content.splitlines(): + op_found = None + op_pos = len(line) + for op in ops: + pos = line.find(op) + if pos >= 0 and pos < op_pos: + op_found = op + op_pos = pos + if not op_found: + continue + (left, value) = map(str.strip, line.split(op_found, 1)) + if value and ((value[0] == '"' and value[-1] == '"') or + (value[0] == "'" and value[-1] == "'")): + value = value[1:-1] + filters_and_key = map(str.strip, left.split(":")) + filters = filters_and_key[:-1] + key = filters_and_key[-1] + for filter in filters: + exp = _get_filter_regex(filter) + if not exp.search(dict["name"]): + break + else: + ops[op_found](dict, key, value) + + + +def _get_filter_regex(filter): + """ + Return a regex object corresponding to a given filter string. + + All regular expressions given to the parser are passed through this + function first. Its purpose is to make them more specific and better + suited to match dictionary names: it forces simple expressions to match + only between dots or at the beginning or end of a string. For example, + the filter 'foo' will match 'foo.bar' but not 'foobar'. + """ + return re.compile(r"(\.|^)(%s)(\.|$)" % filter) + + +def _debug_print(str1, str2=""): + """ + Nicely print two strings and an arrow. + + @param str1: First string. + @param str2: Second string. + """ + if str2: + str = "%-50s ---> %s" % (str1, str2) + else: + str = str1 + logging.debug(str) + + + +# Array structure: +# ---------------- +# The first 4 elements contain the indices of the 4 segments. +# a[0] -- Index of beginning of 'name' segment (always 4). +# a[1] -- Index of beginning of 'shortname' segment. +# a[2] -- Index of beginning of 'depend' segment. +# a[3] -- Index of beginning of 'content' segment. +# The next elements in the array comprise the aforementioned segments: +# The 'name' segment begins with a[a[0]] and ends with a[a[1]-1]. +# The 'shortname' segment begins with a[a[1]] and ends with a[a[2]-1]. +# The 'depend' segment begins with a[a[2]] and ends with a[a[3]-1]. +# The 'content' segment begins with a[a[3]] and ends at the end of the array. + +# The following functions append/prepend to various segments of an array. + +def _array_append_to_name_shortname_depend(a, name, depend): + a.insert(a[1], name) + a.insert(a[2] + 1, name) + a.insert(a[3] + 2, depend) + a[1] += 1 + a[2] += 2 + a[3] += 3 + + +def _array_prepend_to_name_shortname_depend(a, name, depend): + a[1] += 1 + a[2] += 2 + a[3] += 3 + a.insert(a[0], name) + a.insert(a[1], name) + a.insert(a[2], depend) + + +def _array_append_to_name_depend(a, name, depend): + a.insert(a[1], name) + a.insert(a[3] + 1, depend) + a[1] += 1 + a[2] += 1 + a[3] += 2 + + +def _array_prepend_to_name_depend(a, name, depend): + a[1] += 1 + a[2] += 1 + a[3] += 2 + a.insert(a[0], name) + a.insert(a[2], depend) + + +def _array_append_to_content(a, content): + a.append(content) + + +def _array_get_name(a, object_cache): + """ + Return the name of a dictionary represented by a given array. + + @param a: Array representing a dictionary. + @param object_cache: A list of strings referenced by elements in the array. + """ + return ".".join(object_cache[i] for i in a[a[0]:a[1]]) + + +def _array_get_all(a, object_cache): + """ + Return a 4-tuple containing all the data stored in a given array, in a + format that is easy to turn into an actual dictionary. + + @param a: Array representing a dictionary. + @param object_cache: A list of strings referenced by elements in the array. + @return: A 4-tuple: (name, shortname, depend, content), in which all + members are strings except depend which is a list of strings. + """ + name = ".".join(object_cache[i] for i in a[a[0]:a[1]]) + shortname = ".".join(object_cache[i] for i in a[a[1]:a[2]]) + content = "".join(object_cache[i] for i in a[a[3]:]) + depend = [] + prefix = "" + for (n, d) in zip(a[a[0]:a[1]], a[a[2]:a[3]]): + for dep in object_cache[d].split(): + depend.append(prefix + dep) + prefix += object_cache[n] + "." + return (name, shortname, depend, content) + if __name__ == "__main__": parser = optparse.OptionParser() parser.add_option('-f', '--file', dest="filename", action='store', help='path to a config file that will be parsed. ' - 'If not specified, will parse kvm_tests.cfg ' - 'located inside the kvm test dir.') + 'If not specified, will parse tests.cfg located ' + 'inside the kvm test dir.') parser.add_option('--verbose', dest="debug", action='store_true', help='include debug messages in console output') @@ -518,9 +671,9 @@ if __name__ == "__main__": # Here we configure the stand alone program to use the autotest # logging system. logging_manager.configure_logging(KvmLoggingConfig(), verbose=debug) - list = config(filename, debug=debug).get_list() + dicts = config(filename, debug=debug).get_generator() i = 0 - for dict in list: + for dict in dicts: logging.info("Dictionary #%d:", i) keys = dict.keys() keys.sort()