From patchwork Thu Jul 15 15:57:36 2010 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Goldish X-Patchwork-Id: 112264 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter.kernel.org (8.14.4/8.14.3) with ESMTP id o6FFvj1j008921 for ; Thu, 15 Jul 2010 15:57:47 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S933592Ab0GOP5p (ORCPT ); Thu, 15 Jul 2010 11:57:45 -0400 Received: from mx1.redhat.com ([209.132.183.28]:32225 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1754814Ab0GOP5l (ORCPT ); Thu, 15 Jul 2010 11:57:41 -0400 Received: from int-mx04.intmail.prod.int.phx2.redhat.com (int-mx04.intmail.prod.int.phx2.redhat.com [10.5.11.17]) by mx1.redhat.com (8.13.8/8.13.8) with ESMTP id o6FFve9M031881 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=OK); Thu, 15 Jul 2010 11:57:41 -0400 Received: from ns3.rdu.redhat.com (ns3.rdu.redhat.com [10.11.255.199]) by int-mx04.intmail.prod.int.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id o6FFvejs010210; Thu, 15 Jul 2010 11:57:40 -0400 Received: from localhost.localdomain (dhcp-1-188.tlv.redhat.com [10.35.1.188]) by ns3.rdu.redhat.com (8.13.8/8.13.8) with ESMTP id o6FFvUQQ021960; Thu, 15 Jul 2010 11:57:39 -0400 From: Michael Goldish To: autotest@test.kernel.org, kvm@vger.kernel.org Cc: Michael Goldish Subject: [KVM-AUTOTEST PATCH 7/9] [RFC] KVM test: add whql_submission test Date: Thu, 15 Jul 2010 18:57:36 +0300 Message-Id: <1279209458-19590-7-git-send-email-mgoldish@redhat.com> In-Reply-To: <1279209458-19590-6-git-send-email-mgoldish@redhat.com> References: <1279209458-19590-1-git-send-email-mgoldish@redhat.com> <1279209458-19590-2-git-send-email-mgoldish@redhat.com> <1279209458-19590-3-git-send-email-mgoldish@redhat.com> <1279209458-19590-4-git-send-email-mgoldish@redhat.com> <1279209458-19590-5-git-send-email-mgoldish@redhat.com> <1279209458-19590-6-git-send-email-mgoldish@redhat.com> X-Scanned-By: MIMEDefang 2.67 on 10.5.11.17 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.3 (demeter.kernel.org [140.211.167.41]); Thu, 15 Jul 2010 15:57:47 +0000 (UTC) diff --git a/client/tests/kvm/tests/whql_submission.py b/client/tests/kvm/tests/whql_submission.py new file mode 100644 index 0000000..fd238ff --- /dev/null +++ b/client/tests/kvm/tests/whql_submission.py @@ -0,0 +1,176 @@ +import logging, time, os, re +from autotest_lib.client.common_lib import error +import kvm_subprocess, kvm_test_utils, kvm_utils, rss_file_transfer + + +def run_whql_submission(test, params, env): + """ + WHQL submission test: + 1) Log into the guest (the client machine) and into a DTM server machine + 2) Copy the automation program binary (dsso_test_binary) to the server machine + 3) Run the automation program + 4) Pass the program all relevant parameters (e.g. device_data) + 5) Wait for the program to terminate + 6) Parse and report job results + (logs and HTML reports are placed in test.bindir) + + @param test: kvm test object + @param params: Dictionary with the test parameters + @param env: Dictionary with test environment. + """ + vm = kvm_test_utils.get_living_vm(env, params.get("main_vm")) + session = kvm_test_utils.wait_for_login(vm, 0, 240) + + # Collect parameters + server_address = params.get("server_address") + server_shell_port = int(params.get("server_shell_port")) + server_file_transfer_port = int(params.get("server_file_transfer_port")) + server_studio_path = params.get("server_studio_path", "%programfiles%\\ " + "Microsoft Driver Test Manager\\Studio") + dsso_test_binary = params.get("dsso_test_binary", + "deps/whql_submission_15.exe") + dsso_test_binary = kvm_utils.get_path(test.bindir, dsso_test_binary) + test_device = params.get("test_device") + test_timeout = float(params.get("test_timeout", 600)) + wtt_services = params.get("wtt_services") + + # Restart WTT service(s) on the client + logging.info("Restarting WTT services on client") + for svc in wtt_services.split(): + kvm_test_utils.stop_windows_service(session, svc) + for svc in wtt_services.split(): + kvm_test_utils.start_windows_service(session, svc) + + # Copy dsso_test_binary to the server + rss_file_transfer.upload(server_address, server_file_transfer_port, + dsso_test_binary, server_studio_path, timeout=60) + + # Open a shell session with the server + server_session = kvm_utils.remote_login("nc", server_address, + server_shell_port, "", "", + session.prompt, session.linesep) + + # Get the computer names of the server and client + cmd = "echo %computername%" + server_name = server_session.get_command_output(cmd).strip() + client_name = session.get_command_output(cmd).strip() + session.close() + + # Run the automation program on the server + server_session.get_command_output("cd %s" % server_studio_path) + cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary), + server_name, + client_name, + "%s_pool" % client_name, + "%s_submission" % client_name, + test_timeout) + server_session.sendline(cmd) + + # Helper function: wait for a given prompt and raise an exception if an + # error occurs + def find_prompt(prompt): + m, o = server_session.read_until_last_line_matches( + [prompt, server_session.prompt], print_func=logging.info, + timeout=600) + if m != 0: + errors = re.findall("^Error:.*$", o, re.I | re.M) + if errors: + raise error.TestError(errors[0]) + else: + raise error.TestError("Error running automation program: could " + "not find '%s' prompt" % prompt) + + # Tell the automation program which device to test + find_prompt("Device to test:") + server_session.sendline(test_device) + + # Give the automation program all the device data supplied by the user + find_prompt("DeviceData name:") + for dd in kvm_utils.get_sub_dict_names(params, "device_data"): + dd_params = kvm_utils.get_sub_dict(params, dd) + if dd_params.get("dd_name") and dd_params.get("dd_data"): + server_session.sendline(dd_params.get("dd_name")) + server_session.sendline(dd_params.get("dd_data")) + server_session.sendline() + + # Give the automation program all the descriptor information supplied by + # the user + find_prompt("Descriptor path:") + for desc in kvm_utils.get_sub_dict_names(params, "descriptors"): + desc_params = kvm_utils.get_sub_dict(params, desc) + if desc_params.get("desc_path"): + server_session.sendline(desc_params.get("desc_path")) + server_session.sendline() + + # Wait for the automation program to terminate + m, o = server_session.read_up_to_prompt(print_func=logging.info, + timeout=test_timeout + 120) + # (test_timeout + 120 is used here because the automation program is + # supposed to terminate cleanly on its own when test_timeout expires) + server_session.close() + + # Look for test results in the automation program's output + result_summaries = re.findall(r"---- \[.*?\] ----", o, re.DOTALL) + if not result_summaries: + raise error.TestError("The automation program did not return any " + "results") + results = result_summaries[-1].strip("-") + results = eval("".join(results.splitlines())) + + # Download logs and HTML reports from the server + for i, r in enumerate(results): + if "report" in r: + try: + rss_file_transfer.download(server_address, + server_file_transfer_port, + r["report"], test.debugdir) + except rss_file_transfer.FileTransferNotFoundError: + pass + if "logs" in r: + try: + rss_file_transfer.download(server_address, + server_file_transfer_port, + r["logs"], test.debugdir) + except rss_file_transfer.FileTransferNotFoundError: + pass + else: + try: + # Create symlinks to test log dirs to make it easier + # to access them (their original names are not human + # readable) + link_name = "logs_%s" % r["report"].split("\\")[-1] + link_name = link_name.replace(" ", "_") + link_name = link_name.replace("/", "_") + os.symlink(r["logs"].split("\\")[-1], + os.path.join(test.debugdir, link_name)) + except (KeyError, OSError): + pass + + # Print result summary + logging.info("") + logging.info("Result summary:") + name_length = max(len(r.get("job", "")) for r in results) + fmt = "%%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length + logging.info(fmt % ("Job", "Status", "Pass", "Fail", "NotRun", "NotApplicable")) + logging.info(fmt % ("---", "------", "----", "----", "------", "-------------")) + for r in results: + logging.info(fmt % (r.get("job"), r.get("status"), + r.get("pass"), r.get("fail"), r.get("notrun"), + r.get("notapplicable"))) + logging.info("(see logs and HTML reports in %s)" % test.debugdir) + + # Fail if the automation program did not terminate on time + if not m: + raise error.TestFail("The automation program did not terminate " + "on time") + + # Fail if there are failed or incomplete jobs + failed_jobs = [r.get("job") for r in results if r.get("fail", 0) > 0] + running_jobs = [r.get("job") for r in results if r.get("notrun", 0) > 0] + errors = [] + if failed_jobs: + errors += ["Jobs failed: %s." % failed_jobs] + if running_jobs: + errors += ["Jobs did not complete on time: %s." % running_jobs] + if errors: + raise error.TestFail(" ".join(errors))