@@ -797,7 +797,7 @@ def scp_from_remote(host, port, username, password, remote_path, local_path,
def copy_files_to(address, client, username, password, port, local_path,
- remote_path, log_filename=None, timeout=600):
+ remote_path, log_filename=None, verbose=False, timeout=600):
"""
Copy files to a remote host (guest) using the selected client.
@@ -807,22 +807,25 @@ def copy_files_to(address, client, username, password, port, local_path,
@param local_path: Path on the local machine where we are copying from
@param remote_path: Path on the remote machine where we are copying to
@param address: Address of remote host(guest)
- @param log_filename: If specified, log all output to this file
+ @param log_filename: If specified, log all output to this file (SCP only)
+ @param verbose: If True, log some stats using logging.debug (RSS only)
@param timeout: The time duration (in seconds) to wait for the transfer to
- complete.
+ complete.
@raise: Whatever remote_scp() raises
"""
if client == "scp":
scp_to_remote(address, port, username, password, local_path,
remote_path, log_filename, timeout)
elif client == "rss":
- c = rss_file_transfer.FileUploadClient(address, port)
+ log_func = None
+ if verbose: log_func = logging.debug
+ c = rss_file_transfer.FileUploadClient(address, port, log_func)
c.upload(local_path, remote_path, timeout)
c.close()
def copy_files_from(address, client, username, password, port, remote_path,
- local_path, log_filename=None, timeout=600):
+ local_path, log_filename=None, verbose=False, timeout=600):
"""
Copy files from a remote host (guest) using the selected client.
@@ -832,7 +835,8 @@ def copy_files_from(address, client, username, password, port, remote_path,
@param remote_path: Path on the remote machine where we are copying from
@param local_path: Path on the local machine where we are copying to
@param address: Address of remote host(guest)
- @param log_filename: If specified, log all output to this file
+ @param log_filename: If specified, log all output to this file (SCP only)
+ @param verbose: If True, log some stats using logging.debug (RSS only)
@param timeout: The time duration (in seconds) to wait for the transfer to
complete.
@raise: Whatever remote_scp() raises
@@ -841,7 +845,9 @@ def copy_files_from(address, client, username, password, port, remote_path,
scp_from_remote(address, port, username, password, remote_path,
local_path, log_filename, timeout)
elif client == "rss":
- c = rss_file_transfer.FileDownloadClient(address, port)
+ log_func = None
+ if verbose: log_func = logging.debug
+ c = rss_file_transfer.FileDownloadClient(address, port, log_func)
c.download(remote_path, local_path, timeout)
c.close()
@@ -1265,13 +1265,15 @@ class VM:
@error.context_aware
- def copy_files_to(self, host_path, guest_path, nic_index=0, timeout=600):
+ def copy_files_to(self, host_path, guest_path, nic_index=0, verbose=False,
+ timeout=600):
"""
Transfer files to the remote host(guest).
@param host_path: Host path
@param guest_path: Guest path
@param nic_index: The index of the NIC to connect to.
+ @param verbose: If True, log some stats using logging.debug (RSS only)
@param timeout: Time (seconds) before giving up on doing the remote
copy.
"""
@@ -1285,17 +1287,20 @@ class VM:
(self.name, address,
kvm_utils.generate_random_string(4)))
kvm_utils.copy_files_to(address, client, username, password, port,
- host_path, guest_path, log_filename, timeout)
+ host_path, guest_path, log_filename, verbose,
+ timeout)
@error.context_aware
- def copy_files_from(self, guest_path, host_path, nic_index=0, timeout=600):
+ def copy_files_from(self, guest_path, host_path, nic_index=0,
+ verbose=False, timeout=600):
"""
Transfer files from the guest.
@param host_path: Guest path
@param guest_path: Host path
@param nic_index: The index of the NIC to connect to.
+ @param verbose: If True, log some stats using logging.debug (RSS only)
@param timeout: Time (seconds) before giving up on doing the remote
copy.
"""
@@ -1309,7 +1314,8 @@ class VM:
(self.name, address,
kvm_utils.generate_random_string(4)))
kvm_utils.copy_files_from(address, client, username, password, port,
- guest_path, host_path, log_filename, timeout)
+ guest_path, host_path, log_filename,
+ verbose, timeout)
@error.context_aware
@@ -80,12 +80,14 @@ class FileTransferClient(object):
Connect to a RSS (remote shell server) and transfer files.
"""
- def __init__(self, address, port, timeout=20):
+ def __init__(self, address, port, log_func=None, timeout=20):
"""
Connect to a server.
@param address: The server's address
@param port: The server's port
+ @param log_func: If provided, transfer stats will be passed to this
+ function during the transfer
@param timeout: Time duration to wait for connection to succeed
@raise FileTransferConnectError: Raised if the connection fails
"""
@@ -103,6 +105,10 @@ class FileTransferClient(object):
raise FileTransferConnectError("Timeout expired while waiting to "
"receive magic number")
self._send(struct.pack("=i", CHUNKSIZE))
+ self._log_func = log_func
+ self._last_time = time.time()
+ self._last_transferred = 0
+ self.transferred = 0
def __del__(self):
@@ -155,17 +161,37 @@ class FileTransferClient(object):
return "".join(strs)
+ def _report_stats(self, str):
+ if self._log_func:
+ dt = time.time() - self._last_time
+ if dt >= 1:
+ transferred = self.transferred / 1048576.
+ speed = (self.transferred - self._last_transferred) / dt
+ speed /= 1048576.
+ self._log_func("%s %.3f MB (%.3f MB/sec)" %
+ (str, transferred, speed))
+ self._last_time = time.time()
+ self._last_transferred = self.transferred
+
+
def _send_packet(self, str, timeout=60):
self._send(struct.pack("=I", len(str)))
self._send(str, timeout)
+ self.transferred += len(str) + 4
+ self._report_stats("Sent")
def _receive_packet(self, timeout=60):
size = struct.unpack("=I", self._receive(4))[0]
- return self._receive(size, timeout)
+ str = self._receive(size, timeout)
+ self.transferred += len(str) + 4
+ self._report_stats("Received")
+ return str
def _send_file_chunks(self, filename, timeout=60):
+ if self._log_func:
+ self._log_func("Sending file %s" % filename)
f = open(filename, "rb")
try:
try:
@@ -183,6 +209,8 @@ class FileTransferClient(object):
def _receive_file_chunks(self, filename, timeout=60):
+ if self._log_func:
+ self._log_func("Receiving file %s" % filename)
f = open(filename, "wb")
try:
try:
@@ -228,12 +256,14 @@ class FileUploadClient(FileTransferClient):
Connect to a RSS (remote shell server) and upload files or directory trees.
"""
- def __init__(self, address, port, timeout=20):
+ def __init__(self, address, port, log_func=None, timeout=20):
"""
Connect to a server.
@param address: The server's address
@param port: The server's port
+ @param log_func: If provided, transfer stats will be passed to this
+ function during the transfer
@param timeout: Time duration to wait for connection to succeed
@raise FileTransferConnectError: Raised if the connection fails
@raise FileTransferProtocolError: Raised if an incorrect magic number
@@ -241,7 +271,7 @@ class FileUploadClient(FileTransferClient):
@raise FileTransferSocketError: Raised if the RSS_UPLOAD message cannot
be sent to the server
"""
- super(FileUploadClient, self).__init__(address, port, timeout)
+ super(FileUploadClient, self).__init__(address, port, log_func, timeout)
self._send_msg(RSS_UPLOAD)
@@ -325,12 +355,14 @@ class FileDownloadClient(FileTransferClient):
Connect to a RSS (remote shell server) and download files or directory trees.
"""
- def __init__(self, address, port, timeout=20):
+ def __init__(self, address, port, log_func=None, timeout=20):
"""
Connect to a server.
@param address: The server's address
@param port: The server's port
+ @param log_func: If provided, transfer stats will be passed to this
+ function during the transfer
@param timeout: Time duration to wait for connection to succeed
@raise FileTransferConnectError: Raised if the connection fails
@raise FileTransferProtocolError: Raised if an incorrect magic number
@@ -338,7 +370,7 @@ class FileDownloadClient(FileTransferClient):
@raise FileTransferSendError: Raised if the RSS_UPLOAD message cannot
be sent to the server
"""
- super(FileDownloadClient, self).__init__(address, port, timeout)
+ super(FileDownloadClient, self).__init__(address, port, log_func, timeout)
self._send_msg(RSS_DOWNLOAD)
@@ -422,26 +454,26 @@ class FileDownloadClient(FileTransferClient):
raise
-def upload(address, port, src_pattern, dst_path, timeout=60,
+def upload(address, port, src_pattern, dst_path, log_func=None, timeout=60,
connect_timeout=20):
"""
Connect to server and upload files.
@see: FileUploadClient
"""
- client = FileUploadClient(address, port, connect_timeout)
+ client = FileUploadClient(address, port, log_func, connect_timeout)
client.upload(src_pattern, dst_path, timeout)
client.close()
-def download(address, port, src_pattern, dst_path, timeout=60,
+def download(address, port, src_pattern, dst_path, log_func=None, timeout=60,
connect_timeout=20):
"""
Connect to server and upload files.
@see: FileDownloadClient
"""
- client = FileDownloadClient(address, port, connect_timeout)
+ client = FileDownloadClient(address, port, log_func, connect_timeout)
client.download(src_pattern, dst_path, timeout)
client.close()
@@ -457,6 +489,9 @@ def main():
parser.add_option("-u", "--upload",
action="store_true", dest="upload",
help="upload files to server")
+ parser.add_option("-v", "--verbose",
+ action="store_true", dest="verbose",
+ help="be verbose")
parser.add_option("-t", "--timeout",
type="int", dest="timeout", default=3600,
help="transfer timeout")
@@ -468,10 +503,16 @@ def main():
address, port, src_pattern, dst_path = args
port = int(port)
+ logger = None
+ if options.verbose:
+ def p(s):
+ print s
+ logger = p
+
if options.download:
- download(address, port, src_pattern, dst_path, options.timeout)
+ download(address, port, src_pattern, dst_path, logger, options.timeout)
elif options.upload:
- upload(address, port, src_pattern, dst_path, options.timeout)
+ upload(address, port, src_pattern, dst_path, logger, options.timeout)
if __name__ == "__main__":
@@ -51,15 +51,15 @@ def run_migration_with_file_transfer(test, params, env):
error.context("transferring file to guest while migrating",
logging.info)
- bg = kvm_utils.Thread(vm.copy_files_to,
- (host_path, guest_path, 0, transfer_timeout))
+ bg = kvm_utils.Thread(vm.copy_files_to, (host_path, guest_path),
+ dict(verbose=True, timeout=transfer_timeout))
run_and_migrate(bg)
error.context("transferring file back to host while migrating",
logging.info)
bg = kvm_utils.Thread(vm.copy_files_from,
- (guest_path, host_path_returned, 0,
- transfer_timeout))
+ (guest_path, host_path_returned),
+ dict(verbose=True, timeout=transfer_timeout))
run_and_migrate(bg)
# Make sure the returned file is identical to the original one
@@ -35,8 +35,8 @@ def run_vmstop(test, params, env):
utils.run("dd if=/dev/zero of=/tmp/file bs=1M count=%s" % file_size)
# Transfer file from host to guest, we didn't expect the finish of
# transfer, we just let it to be a kind of stress in guest.
- bg = kvm_utils.Thread(vm.copy_files_to, ("/tmp/file",
- guest_path, 0, 60))
+ bg = kvm_utils.Thread(vm.copy_files_to, ("/tmp/file", guest_path),
+ dict(verbose=True, timeout=60))
logging.info("Start the background transfer")
bg.start()