From patchwork Sun Sep 20 15:16:30 2009 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Goldish X-Patchwork-Id: 48906 Received: from vger.kernel.org (vger.kernel.org [209.132.176.167]) by demeter.kernel.org (8.14.2/8.14.2) with ESMTP id n8KFJUQ1015083 for ; Sun, 20 Sep 2009 15:19:30 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754014AbZITPTZ (ORCPT ); Sun, 20 Sep 2009 11:19:25 -0400 Received: (majordomo@vger.kernel.org) by vger.kernel.org id S1753890AbZITPTY (ORCPT ); Sun, 20 Sep 2009 11:19:24 -0400 Received: from mx1.redhat.com ([209.132.183.28]:8461 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1753988AbZITPTX (ORCPT ); Sun, 20 Sep 2009 11:19:23 -0400 Received: from int-mx03.intmail.prod.int.phx2.redhat.com (int-mx03.intmail.prod.int.phx2.redhat.com [10.5.11.16]) by mx1.redhat.com (8.13.8/8.13.8) with ESMTP id n8KFJQl4007933; Sun, 20 Sep 2009 11:19:26 -0400 Received: from ns3.rdu.redhat.com (ns3.rdu.redhat.com [10.11.255.199]) by int-mx03.intmail.prod.int.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id n8KFJPab004801; Sun, 20 Sep 2009 11:19:25 -0400 Received: from localhost.localdomain (dhcp-1-188.tlv.redhat.com [10.35.1.188]) by ns3.rdu.redhat.com (8.13.8/8.13.8) with ESMTP id n8KFJJCa001335; Sun, 20 Sep 2009 11:19:24 -0400 From: Michael Goldish To: autotest@test.kernel.org, kvm@vger.kernel.org Cc: Michael Goldish Subject: [KVM-AUTOTEST PATCH 4/4] KVM test: kvm_subprocess: minimize the number of modules imported by the server Date: Sun, 20 Sep 2009 18:16:30 +0300 Message-Id: <1253459790-17859-4-git-send-email-mgoldish@redhat.com> In-Reply-To: <1253459790-17859-3-git-send-email-mgoldish@redhat.com> References: <1253459790-17859-1-git-send-email-mgoldish@redhat.com> <1253459790-17859-2-git-send-email-mgoldish@redhat.com> <1253459790-17859-3-git-send-email-mgoldish@redhat.com> X-Scanned-By: MIMEDefang 2.67 on 10.5.11.16 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org Rearrange the code (put the server on top) so that the server imports only the modules it requires. This reduces the startup time of child processes. Signed-off-by: Michael Goldish --- client/tests/kvm/kvm_subprocess.py | 359 ++++++++++++++++++------------------ 1 files changed, 181 insertions(+), 178 deletions(-) diff --git a/client/tests/kvm/kvm_subprocess.py b/client/tests/kvm/kvm_subprocess.py index f748586..424c801 100755 --- a/client/tests/kvm/kvm_subprocess.py +++ b/client/tests/kvm/kvm_subprocess.py @@ -5,8 +5,187 @@ A class and functions used for running and controlling child processes. @copyright: 2008-2009 Red Hat Inc. """ -import sys, subprocess, pty, select, os, time, signal, re, termios, fcntl -import threading, logging, commands +import os, sys, pty, select, termios, fcntl + + +# The following helper functions are shared by the server and the client. + +def _lock(filename): + if not os.path.exists(filename): + open(filename, "w").close() + fd = os.open(filename, os.O_RDWR) + fcntl.lockf(fd, fcntl.LOCK_EX) + return fd + + +def _unlock(fd): + fcntl.lockf(fd, fcntl.LOCK_UN) + os.close(fd) + + +def _locked(filename): + try: + fd = os.open(filename, os.O_RDWR) + except: + return False + try: + fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except: + os.close(fd) + return True + fcntl.lockf(fd, fcntl.LOCK_UN) + os.close(fd) + return False + + +def _wait(filename): + fd = _lock(filename) + _unlock(fd) + + +def _get_filenames(base_dir, id): + return [os.path.join(base_dir, s + id) for s in + "shell-pid-", "status-", "output-", "inpipe-", + "lock-server-running-", "lock-client-starting-"] + + +def _get_reader_filename(base_dir, id, reader): + return os.path.join(base_dir, "outpipe-%s-%s" % (reader, id)) + + +# The following is the server part of the module. + +if __name__ == "__main__": + id = sys.stdin.readline().strip() + echo = sys.stdin.readline().strip() == "True" + readers = sys.stdin.readline().strip().split(",") + command = sys.stdin.readline().strip() + " && echo %s > /dev/null" % id + + # Define filenames to be used for communication + base_dir = "/tmp/kvm_spawn" + (shell_pid_filename, + status_filename, + output_filename, + inpipe_filename, + lock_server_running_filename, + lock_client_starting_filename) = _get_filenames(base_dir, id) + + # Populate the reader filenames list + reader_filenames = [_get_reader_filename(base_dir, id, reader) + for reader in readers] + + # Set $TERM = dumb + os.putenv("TERM", "dumb") + + (shell_pid, shell_fd) = pty.fork() + if shell_pid == 0: + # Child process: run the command in a subshell + os.execv("/bin/sh", ["/bin/sh", "-c", command]) + else: + # Parent process + lock_server_running = _lock(lock_server_running_filename) + + # Set terminal echo on/off and disable pre- and post-processing + attr = termios.tcgetattr(shell_fd) + attr[0] &= ~termios.INLCR + attr[0] &= ~termios.ICRNL + attr[0] &= ~termios.IGNCR + attr[1] &= ~termios.OPOST + if echo: + attr[3] |= termios.ECHO + else: + attr[3] &= ~termios.ECHO + termios.tcsetattr(shell_fd, termios.TCSANOW, attr) + + # Open output file + output_file = open(output_filename, "w") + # Open input pipe + os.mkfifo(inpipe_filename) + inpipe_fd = os.open(inpipe_filename, os.O_RDWR) + # Open output pipes (readers) + reader_fds = [] + for filename in reader_filenames: + os.mkfifo(filename) + reader_fds.append(os.open(filename, os.O_RDWR)) + + # Write shell PID to file + file = open(shell_pid_filename, "w") + file.write(str(shell_pid)) + file.close() + + # Print something to stdout so the client can start working + print "Server %s ready" % id + sys.stdout.flush() + + # Initialize buffers + buffers = ["" for reader in readers] + + # Read from child and write to files/pipes + while True: + check_termination = False + # Make a list of reader pipes whose buffers are not empty + fds = [fd for (i, fd) in enumerate(reader_fds) if buffers[i]] + # Wait until there's something to do + r, w, x = select.select([shell_fd, inpipe_fd], fds, [], 0.5) + # If a reader pipe is ready for writing -- + for (i, fd) in enumerate(reader_fds): + if fd in w: + bytes_written = os.write(fd, buffers[i]) + buffers[i] = buffers[i][bytes_written:] + # If there's data to read from the child process -- + if shell_fd in r: + try: + data = os.read(shell_fd, 16384) + except OSError: + data = "" + if not data: + check_termination = True + # Remove carriage returns from the data -- they often cause + # trouble and are normally not needed + data = data.replace("\r", "") + output_file.write(data) + output_file.flush() + for i in range(len(readers)): + buffers[i] += data + # If os.read() raised an exception or there was nothing to read -- + if check_termination or shell_fd not in r: + pid, status = os.waitpid(shell_pid, os.WNOHANG) + if pid: + status = os.WEXITSTATUS(status) + break + # If there's data to read from the client -- + if inpipe_fd in r: + data = os.read(inpipe_fd, 1024) + os.write(shell_fd, data) + + # Write the exit status to a file + file = open(status_filename, "w") + file.write(str(status)) + file.close() + + # Wait for the client to finish initializing + _wait(lock_client_starting_filename) + + # Delete FIFOs + for filename in reader_filenames + [inpipe_filename]: + try: + os.unlink(filename) + except OSError: + pass + + # Close all files and pipes + output_file.close() + os.close(inpipe_fd) + for fd in reader_fds: + os.close(fd) + + _unlock(lock_server_running) + exit(0) + + +# The following is the client part of the module. + +import subprocess, time, signal, re, threading, logging import common, kvm_utils @@ -77,49 +256,6 @@ def run_fg(command, output_func=None, output_prefix="", timeout=1.0): return (status, output) -def _lock(filename): - if not os.path.exists(filename): - open(filename, "w").close() - fd = os.open(filename, os.O_RDWR) - fcntl.lockf(fd, fcntl.LOCK_EX) - return fd - - -def _unlock(fd): - fcntl.lockf(fd, fcntl.LOCK_UN) - os.close(fd) - - -def _locked(filename): - try: - fd = os.open(filename, os.O_RDWR) - except: - return False - try: - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except: - os.close(fd) - return True - fcntl.lockf(fd, fcntl.LOCK_UN) - os.close(fd) - return False - - -def _wait(filename): - fd = _lock(filename) - _unlock(fd) - - -def _get_filenames(base_dir, id): - return [os.path.join(base_dir, s + id) for s in - "shell-pid-", "status-", "output-", "inpipe-", - "lock-server-running-", "lock-client-starting-"] - - -def _get_reader_filename(base_dir, id, reader): - return os.path.join(base_dir, "outpipe-%s-%s" % (reader, id)) - - class kvm_spawn: """ This class is used for spawning and controlling a child process. @@ -1031,136 +1167,3 @@ class kvm_shell_session(kvm_expect): internal_timeout, print_func) return output - - -# The following is the server part of the module. - -def _server_main(): - id = sys.stdin.readline().strip() - echo = sys.stdin.readline().strip() == "True" - readers = sys.stdin.readline().strip().split(",") - command = sys.stdin.readline().strip() + " && echo %s > /dev/null" % id - - # Define filenames to be used for communication - base_dir = "/tmp/kvm_spawn" - (shell_pid_filename, - status_filename, - output_filename, - inpipe_filename, - lock_server_running_filename, - lock_client_starting_filename) = _get_filenames(base_dir, id) - - # Populate the reader filenames list - reader_filenames = [_get_reader_filename(base_dir, id, reader) - for reader in readers] - - # Set $TERM = dumb - os.putenv("TERM", "dumb") - - (shell_pid, shell_fd) = pty.fork() - if shell_pid == 0: - # Child process: run the command in a subshell - os.execv("/bin/sh", ["/bin/sh", "-c", command]) - else: - # Parent process - lock_server_running = _lock(lock_server_running_filename) - - # Set terminal echo on/off and disable pre- and post-processing - attr = termios.tcgetattr(shell_fd) - attr[0] &= ~termios.INLCR - attr[0] &= ~termios.ICRNL - attr[0] &= ~termios.IGNCR - attr[1] &= ~termios.OPOST - if echo: - attr[3] |= termios.ECHO - else: - attr[3] &= ~termios.ECHO - termios.tcsetattr(shell_fd, termios.TCSANOW, attr) - - # Open output file - output_file = open(output_filename, "w") - # Open input pipe - os.mkfifo(inpipe_filename) - inpipe_fd = os.open(inpipe_filename, os.O_RDWR) - # Open output pipes (readers) - reader_fds = [] - for filename in reader_filenames: - os.mkfifo(filename) - reader_fds.append(os.open(filename, os.O_RDWR)) - - # Write shell PID to file - file = open(shell_pid_filename, "w") - file.write(str(shell_pid)) - file.close() - - # Print something to stdout so the client can start working - print "Server %s ready" % id - sys.stdout.flush() - - # Initialize buffers - buffers = ["" for reader in readers] - - # Read from child and write to files/pipes - while True: - check_termination = False - # Make a list of reader pipes whose buffers are not empty - fds = [fd for (i, fd) in enumerate(reader_fds) if buffers[i]] - # Wait until there's something to do - r, w, x = select.select([shell_fd, inpipe_fd], fds, [], 0.5) - # If a reader pipe is ready for writing -- - for (i, fd) in enumerate(reader_fds): - if fd in w: - bytes_written = os.write(fd, buffers[i]) - buffers[i] = buffers[i][bytes_written:] - # If there's data to read from the child process -- - if shell_fd in r: - try: - data = os.read(shell_fd, 16384) - except OSError: - data = "" - if not data: - check_termination = True - # Remove carriage returns from the data -- they often cause - # trouble and are normally not needed - data = data.replace("\r", "") - output_file.write(data) - output_file.flush() - for i in range(len(readers)): - buffers[i] += data - # If os.read() raised an exception or there was nothing to read -- - if check_termination or shell_fd not in r: - pid, status = os.waitpid(shell_pid, os.WNOHANG) - if pid: - status = os.WEXITSTATUS(status) - break - # If there's data to read from the client -- - if inpipe_fd in r: - data = os.read(inpipe_fd, 1024) - os.write(shell_fd, data) - - # Write the exit status to a file - file = open(status_filename, "w") - file.write(str(status)) - file.close() - - # Wait for the client to finish initializing - _wait(lock_client_starting_filename) - - # Delete FIFOs - for filename in reader_filenames + [inpipe_filename]: - try: - os.unlink(filename) - except OSError: - pass - - # Close all files and pipes - output_file.close() - os.close(inpipe_fd) - for fd in reader_fds: - os.close(fd) - - _unlock(lock_server_running) - - -if __name__ == "__main__": - _server_main()