From patchwork Tue Jan 11 13:13:34 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Goldish X-Patchwork-Id: 471631 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter1.kernel.org (8.14.4/8.14.3) with ESMTP id p0BDEHT9008472 for ; Tue, 11 Jan 2011 13:14:25 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1755728Ab1AKNNq (ORCPT ); Tue, 11 Jan 2011 08:13:46 -0500 Received: from mx1.redhat.com ([209.132.183.28]:17922 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1756035Ab1AKNNm (ORCPT ); Tue, 11 Jan 2011 08:13:42 -0500 Received: from int-mx01.intmail.prod.int.phx2.redhat.com (int-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.11]) by mx1.redhat.com (8.13.8/8.13.8) with ESMTP id p0BDDfMW013620 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=OK); Tue, 11 Jan 2011 08:13:41 -0500 Received: from ns3.rdu.redhat.com (ns3.rdu.redhat.com [10.11.255.199]) by int-mx01.intmail.prod.int.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id p0BDDfdG022804; Tue, 11 Jan 2011 08:13:41 -0500 Received: from qu0061.eng.lab.tlv.redhat.com ([10.35.16.61]) by ns3.rdu.redhat.com (8.13.8/8.13.8) with ESMTP id p0BDD9ni025030; Tue, 11 Jan 2011 08:13:39 -0500 From: Michael Goldish To: autotest@test.kernel.org, kvm@vger.kernel.org Cc: Michael Goldish Subject: [KVM-AUTOTEST PATCH 22/26] KVM test: rss_file_transfer: optionally log some statistics Date: Tue, 11 Jan 2011 15:13:34 +0200 Message-Id: <1294751618-21631-22-git-send-email-mgoldish@redhat.com> In-Reply-To: <1294751618-21631-1-git-send-email-mgoldish@redhat.com> References: <1294751618-21631-1-git-send-email-mgoldish@redhat.com> X-Scanned-By: MIMEDefang 2.67 on 10.5.11.11 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.6 (demeter1.kernel.org [140.211.167.41]); Tue, 11 Jan 2011 13:14:25 +0000 (UTC) diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py index d7e205d..8e6cef1 100644 --- a/client/tests/kvm/kvm_utils.py +++ b/client/tests/kvm/kvm_utils.py @@ -797,7 +797,7 @@ def scp_from_remote(host, port, username, password, remote_path, local_path, def copy_files_to(address, client, username, password, port, local_path, - remote_path, log_filename=None, timeout=600): + remote_path, log_filename=None, verbose=False, timeout=600): """ Copy files to a remote host (guest) using the selected client. @@ -807,22 +807,25 @@ def copy_files_to(address, client, username, password, port, local_path, @param local_path: Path on the local machine where we are copying from @param remote_path: Path on the remote machine where we are copying to @param address: Address of remote host(guest) - @param log_filename: If specified, log all output to this file + @param log_filename: If specified, log all output to this file (SCP only) + @param verbose: If True, log some stats using logging.debug (RSS only) @param timeout: The time duration (in seconds) to wait for the transfer to - complete. + complete. @raise: Whatever remote_scp() raises """ if client == "scp": scp_to_remote(address, port, username, password, local_path, remote_path, log_filename, timeout) elif client == "rss": - c = rss_file_transfer.FileUploadClient(address, port) + log_func = None + if verbose: log_func = logging.debug + c = rss_file_transfer.FileUploadClient(address, port, log_func) c.upload(local_path, remote_path, timeout) c.close() def copy_files_from(address, client, username, password, port, remote_path, - local_path, log_filename=None, timeout=600): + local_path, log_filename=None, verbose=False, timeout=600): """ Copy files from a remote host (guest) using the selected client. @@ -832,7 +835,8 @@ def copy_files_from(address, client, username, password, port, remote_path, @param remote_path: Path on the remote machine where we are copying from @param local_path: Path on the local machine where we are copying to @param address: Address of remote host(guest) - @param log_filename: If specified, log all output to this file + @param log_filename: If specified, log all output to this file (SCP only) + @param verbose: If True, log some stats using logging.debug (RSS only) @param timeout: The time duration (in seconds) to wait for the transfer to complete. @raise: Whatever remote_scp() raises @@ -841,7 +845,9 @@ def copy_files_from(address, client, username, password, port, remote_path, scp_from_remote(address, port, username, password, remote_path, local_path, log_filename, timeout) elif client == "rss": - c = rss_file_transfer.FileDownloadClient(address, port) + log_func = None + if verbose: log_func = logging.debug + c = rss_file_transfer.FileDownloadClient(address, port, log_func) c.download(remote_path, local_path, timeout) c.close() diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py index 1fcb352..0c355da 100755 --- a/client/tests/kvm/kvm_vm.py +++ b/client/tests/kvm/kvm_vm.py @@ -1265,13 +1265,15 @@ class VM: @error.context_aware - def copy_files_to(self, host_path, guest_path, nic_index=0, timeout=600): + def copy_files_to(self, host_path, guest_path, nic_index=0, verbose=False, + timeout=600): """ Transfer files to the remote host(guest). @param host_path: Host path @param guest_path: Guest path @param nic_index: The index of the NIC to connect to. + @param verbose: If True, log some stats using logging.debug (RSS only) @param timeout: Time (seconds) before giving up on doing the remote copy. """ @@ -1285,17 +1287,20 @@ class VM: (self.name, address, kvm_utils.generate_random_string(4))) kvm_utils.copy_files_to(address, client, username, password, port, - host_path, guest_path, log_filename, timeout) + host_path, guest_path, log_filename, verbose, + timeout) @error.context_aware - def copy_files_from(self, guest_path, host_path, nic_index=0, timeout=600): + def copy_files_from(self, guest_path, host_path, nic_index=0, + verbose=False, timeout=600): """ Transfer files from the guest. @param host_path: Guest path @param guest_path: Host path @param nic_index: The index of the NIC to connect to. + @param verbose: If True, log some stats using logging.debug (RSS only) @param timeout: Time (seconds) before giving up on doing the remote copy. """ @@ -1309,7 +1314,8 @@ class VM: (self.name, address, kvm_utils.generate_random_string(4))) kvm_utils.copy_files_from(address, client, username, password, port, - guest_path, host_path, log_filename, timeout) + guest_path, host_path, log_filename, + verbose, timeout) @error.context_aware diff --git a/client/tests/kvm/rss_file_transfer.py b/client/tests/kvm/rss_file_transfer.py index d907678..4d00d17 100755 --- a/client/tests/kvm/rss_file_transfer.py +++ b/client/tests/kvm/rss_file_transfer.py @@ -80,12 +80,14 @@ class FileTransferClient(object): Connect to a RSS (remote shell server) and transfer files. """ - def __init__(self, address, port, timeout=20): + def __init__(self, address, port, log_func=None, timeout=20): """ Connect to a server. @param address: The server's address @param port: The server's port + @param log_func: If provided, transfer stats will be passed to this + function during the transfer @param timeout: Time duration to wait for connection to succeed @raise FileTransferConnectError: Raised if the connection fails """ @@ -103,6 +105,10 @@ class FileTransferClient(object): raise FileTransferConnectError("Timeout expired while waiting to " "receive magic number") self._send(struct.pack("=i", CHUNKSIZE)) + self._log_func = log_func + self._last_time = time.time() + self._last_transferred = 0 + self.transferred = 0 def __del__(self): @@ -155,17 +161,37 @@ class FileTransferClient(object): return "".join(strs) + def _report_stats(self, str): + if self._log_func: + dt = time.time() - self._last_time + if dt >= 1: + transferred = self.transferred / 1048576. + speed = (self.transferred - self._last_transferred) / dt + speed /= 1048576. + self._log_func("%s %.3f MB (%.3f MB/sec)" % + (str, transferred, speed)) + self._last_time = time.time() + self._last_transferred = self.transferred + + def _send_packet(self, str, timeout=60): self._send(struct.pack("=I", len(str))) self._send(str, timeout) + self.transferred += len(str) + 4 + self._report_stats("Sent") def _receive_packet(self, timeout=60): size = struct.unpack("=I", self._receive(4))[0] - return self._receive(size, timeout) + str = self._receive(size, timeout) + self.transferred += len(str) + 4 + self._report_stats("Received") + return str def _send_file_chunks(self, filename, timeout=60): + if self._log_func: + self._log_func("Sending file %s" % filename) f = open(filename, "rb") try: try: @@ -183,6 +209,8 @@ class FileTransferClient(object): def _receive_file_chunks(self, filename, timeout=60): + if self._log_func: + self._log_func("Receiving file %s" % filename) f = open(filename, "wb") try: try: @@ -228,12 +256,14 @@ class FileUploadClient(FileTransferClient): Connect to a RSS (remote shell server) and upload files or directory trees. """ - def __init__(self, address, port, timeout=20): + def __init__(self, address, port, log_func=None, timeout=20): """ Connect to a server. @param address: The server's address @param port: The server's port + @param log_func: If provided, transfer stats will be passed to this + function during the transfer @param timeout: Time duration to wait for connection to succeed @raise FileTransferConnectError: Raised if the connection fails @raise FileTransferProtocolError: Raised if an incorrect magic number @@ -241,7 +271,7 @@ class FileUploadClient(FileTransferClient): @raise FileTransferSocketError: Raised if the RSS_UPLOAD message cannot be sent to the server """ - super(FileUploadClient, self).__init__(address, port, timeout) + super(FileUploadClient, self).__init__(address, port, log_func, timeout) self._send_msg(RSS_UPLOAD) @@ -325,12 +355,14 @@ class FileDownloadClient(FileTransferClient): Connect to a RSS (remote shell server) and download files or directory trees. """ - def __init__(self, address, port, timeout=20): + def __init__(self, address, port, log_func=None, timeout=20): """ Connect to a server. @param address: The server's address @param port: The server's port + @param log_func: If provided, transfer stats will be passed to this + function during the transfer @param timeout: Time duration to wait for connection to succeed @raise FileTransferConnectError: Raised if the connection fails @raise FileTransferProtocolError: Raised if an incorrect magic number @@ -338,7 +370,7 @@ class FileDownloadClient(FileTransferClient): @raise FileTransferSendError: Raised if the RSS_UPLOAD message cannot be sent to the server """ - super(FileDownloadClient, self).__init__(address, port, timeout) + super(FileDownloadClient, self).__init__(address, port, log_func, timeout) self._send_msg(RSS_DOWNLOAD) @@ -422,26 +454,26 @@ class FileDownloadClient(FileTransferClient): raise -def upload(address, port, src_pattern, dst_path, timeout=60, +def upload(address, port, src_pattern, dst_path, log_func=None, timeout=60, connect_timeout=20): """ Connect to server and upload files. @see: FileUploadClient """ - client = FileUploadClient(address, port, connect_timeout) + client = FileUploadClient(address, port, log_func, connect_timeout) client.upload(src_pattern, dst_path, timeout) client.close() -def download(address, port, src_pattern, dst_path, timeout=60, +def download(address, port, src_pattern, dst_path, log_func=None, timeout=60, connect_timeout=20): """ Connect to server and upload files. @see: FileDownloadClient """ - client = FileDownloadClient(address, port, connect_timeout) + client = FileDownloadClient(address, port, log_func, connect_timeout) client.download(src_pattern, dst_path, timeout) client.close() @@ -457,6 +489,9 @@ def main(): parser.add_option("-u", "--upload", action="store_true", dest="upload", help="upload files to server") + parser.add_option("-v", "--verbose", + action="store_true", dest="verbose", + help="be verbose") parser.add_option("-t", "--timeout", type="int", dest="timeout", default=3600, help="transfer timeout") @@ -468,10 +503,16 @@ def main(): address, port, src_pattern, dst_path = args port = int(port) + logger = None + if options.verbose: + def p(s): + print s + logger = p + if options.download: - download(address, port, src_pattern, dst_path, options.timeout) + download(address, port, src_pattern, dst_path, logger, options.timeout) elif options.upload: - upload(address, port, src_pattern, dst_path, options.timeout) + upload(address, port, src_pattern, dst_path, logger, options.timeout) if __name__ == "__main__": diff --git a/client/tests/kvm/tests/migration_with_file_transfer.py b/client/tests/kvm/tests/migration_with_file_transfer.py index 58601b4..4064b4a 100644 --- a/client/tests/kvm/tests/migration_with_file_transfer.py +++ b/client/tests/kvm/tests/migration_with_file_transfer.py @@ -51,15 +51,15 @@ def run_migration_with_file_transfer(test, params, env): error.context("transferring file to guest while migrating", logging.info) - bg = kvm_utils.Thread(vm.copy_files_to, - (host_path, guest_path, 0, transfer_timeout)) + bg = kvm_utils.Thread(vm.copy_files_to, (host_path, guest_path), + dict(verbose=True, timeout=transfer_timeout)) run_and_migrate(bg) error.context("transferring file back to host while migrating", logging.info) bg = kvm_utils.Thread(vm.copy_files_from, - (guest_path, host_path_returned, 0, - transfer_timeout)) + (guest_path, host_path_returned), + dict(verbose=True, timeout=transfer_timeout)) run_and_migrate(bg) # Make sure the returned file is identical to the original one diff --git a/client/tests/kvm/tests/vmstop.py b/client/tests/kvm/tests/vmstop.py index 5bff3b3..0fb48fa 100644 --- a/client/tests/kvm/tests/vmstop.py +++ b/client/tests/kvm/tests/vmstop.py @@ -35,8 +35,8 @@ def run_vmstop(test, params, env): utils.run("dd if=/dev/zero of=/tmp/file bs=1M count=%s" % file_size) # Transfer file from host to guest, we didn't expect the finish of # transfer, we just let it to be a kind of stress in guest. - bg = kvm_utils.Thread(vm.copy_files_to, ("/tmp/file", - guest_path, 0, 60)) + bg = kvm_utils.Thread(vm.copy_files_to, ("/tmp/file", guest_path), + dict(verbose=True, timeout=60)) logging.info("Start the background transfer") bg.start()