From patchwork Tue Sep 15 02:20:35 2009 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yolkfull Chow X-Patchwork-Id: 47563 Received: from vger.kernel.org (vger.kernel.org [209.132.176.167]) by demeter.kernel.org (8.14.2/8.14.2) with ESMTP id n8F2KAka028279 for ; Tue, 15 Sep 2009 02:20:10 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1757341AbZIOCUD (ORCPT ); Mon, 14 Sep 2009 22:20:03 -0400 Received: (majordomo@vger.kernel.org) by vger.kernel.org id S1757314AbZIOCUD (ORCPT ); Mon, 14 Sep 2009 22:20:03 -0400 Received: from mx1.redhat.com ([209.132.183.28]:13752 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1757171AbZIOCUB (ORCPT ); Mon, 14 Sep 2009 22:20:01 -0400 Received: from int-mx03.intmail.prod.int.phx2.redhat.com (int-mx03.intmail.prod.int.phx2.redhat.com [10.5.11.16]) by mx1.redhat.com (8.13.8/8.13.8) with ESMTP id n8F2K3gR005875; Mon, 14 Sep 2009 22:20:03 -0400 Received: from localhost.localdomain (dhcp-65-181.nay.redhat.com [10.66.65.181]) by int-mx03.intmail.prod.int.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id n8F2K0Sm025052; Mon, 14 Sep 2009 22:20:01 -0400 From: Yolkfull Chow To: kvm@vger.kernel.org Cc: autotest@test.kernel.org, Yolkfull Chow Subject: [PATCH] Add pass through feature test (support SR-IOV) Date: Tue, 15 Sep 2009 10:20:35 +0800 Message-Id: <1252981235-4145-1-git-send-email-yzhou@redhat.com> X-Scanned-By: MIMEDefang 2.67 on 10.5.11.16 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org It supports both SR-IOV virtual functions' and physical NIC card pass through. * For SR-IOV virtual functions passthrough, we could specify the module parameter 'max_vfs' in config file. * For physical NIC card pass through, we should specify the device name(s). Signed-off-by: Yolkfull Chow --- client/tests/kvm/kvm_tests.cfg.sample | 12 ++ client/tests/kvm/kvm_utils.py | 248 ++++++++++++++++++++++++++++++++- client/tests/kvm/kvm_vm.py | 68 +++++++++- 3 files changed, 326 insertions(+), 2 deletions(-) diff --git a/client/tests/kvm/kvm_tests.cfg.sample b/client/tests/kvm/kvm_tests.cfg.sample index a83ef9b..c6037da 100644 --- a/client/tests/kvm/kvm_tests.cfg.sample +++ b/client/tests/kvm/kvm_tests.cfg.sample @@ -627,6 +627,18 @@ variants: variants: + - @no_passthrough: + pass_through = no + - nic_passthrough: + pass_through = pf + passthrough_devs = eth1 + - vfs_passthrough: + pass_through = vf + max_vfs = 7 + vfs_count = 7 + + +variants: - @basic: only Fedora Windows - @full: diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py index dfca938..1fe3b31 100644 --- a/client/tests/kvm/kvm_utils.py +++ b/client/tests/kvm/kvm_utils.py @@ -1,5 +1,5 @@ import md5, thread, subprocess, time, string, random, socket, os, signal, pty -import select, re, logging, commands +import select, re, logging, commands, cPickle from autotest_lib.client.bin import utils from autotest_lib.client.common_lib import error import kvm_subprocess @@ -795,3 +795,249 @@ def md5sum_file(filename, size=None): size -= len(data) f.close() return o.hexdigest() + + +def get_full_id(pci_id): + """ + Get full PCI ID of pci_id. + """ + cmd = "lspci -D | awk '/%s/ {print $1}'" % pci_id + status, full_id = commands.getstatusoutput(cmd) + if status != 0: + return None + return full_id + + +def get_vendor_id(pci_id): + """ + Check out the device vendor ID according to PCI ID. + """ + cmd = "lspci -n | awk '/%s/ {print $3}'" % pci_id + return re.sub(":", " ", commands.getoutput(cmd)) + + +def release_pci_devs(dict): + """ + Release assigned PCI devices to host. + """ + def release_dev(pci_id): + base_dir = "/sys/bus/pci" + full_id = get_full_id(pci_id) + vendor_id = get_vendor_id(pci_id) + drv_path = os.path.join(base_dir, "devices/%s/driver" % full_id) + if 'pci-stub' in os.readlink(drv_path): + cmd = "echo '%s' > %s/new_id" % (vendor_id, drv_path) + if os.system(cmd): + return False + + stub_path = os.path.join(base_dir, "drivers/pci-stub") + cmd = "echo '%s' > %s/unbind" % (full_id, stub_path) + if os.system(cmd): + return False + + prev_driver = self.dev_prev_drivers[pci_id] + cmd = "echo '%s' > %s/bind" % (full_id, prev_driver) + if os.system(cmd): + return False + return True + + for pci_id in dict.keys(): + if not release_dev(pci_id): + logging.error("Failed to release device [%s] to host" % pci_id) + else: + logging.info("Release device [%s] successfully" % pci_id) + + +class PassThrough: + """ + Request passthroughable devices on host. It will check whether to request + PF(physical NIC cards) or VF(Virtual Functions). + """ + def __init__(self, type="nic_vf", max_vfs=None, names=None): + """ + Initialize parameter 'type' which could be: + nic_vf: Virtual Functions + nic_pf: Physical NIC card + mixed: Both includes VFs and PFs + + If pass through Physical NIC cards, we need to specify which devices + to be assigned, e.g. 'eth1 eth2'. + + If pass through Virtual Functions, we need to specify how many vfs + are going to be assigned, e.g. passthrough_count = 8 and max_vfs in + config file. + + @param type: Pass through device's type + @param max_vfs: parameter of module 'igb' + @param names: Physical NIC cards' names, e.g.'eth1 eth2 ...' + """ + self.type = type + if max_vfs: + self.max_vfs = int(max_vfs) + if names: + self.name_list = names.split() + + def sr_iov_setup(self): + """ + Setup SR-IOV environment, check if module 'igb' is loaded with + parameter 'max_vfs'. + """ + re_probe = False + # Check whether the module 'igb' is loaded + s, o = commands.getstatusoutput('lsmod | grep igb') + if s: + re_probe = True + elif not self.chk_vfs_count(): + os.system("modprobe -r igb") + re_probe = True + + # Re-probe module 'igb' + if re_probe: + cmd = "modprobe igb max_vfs=%d" % self.max_vfs + s, o = commands.getstatusoutput(cmd) + if s: + return False + if not self.chk_vfs_count(): + return False + return True + + def get_vfs_count(self): + """ + Get Vfs count number according to 'max_vfs' + """ + cmd = "lspci | grep 'Virtual Function' | wc -l" + return int(commands.getoutput(cmd)) + + def chk_vfs_count(self): + """ + Check VFs count number according to parameter 'max_vfs'. + """ + if self.get_vfs_count() != (self.max_vfs * 2): + return False + return True + + def _get_pci_id(self, name, search_str): + """ + Get the device's PCI ID according to name. + """ + cmd = "ethtool -i %s | awk '/bus-info/ {print $2}'" % name + s, pci_id = commands.getstatusoutput(cmd) + if not (s or "Cannot get driver information" in pci_id): + return pci_id[5:] + + cmd = "lspci | awk '/%s/ {print $1}'" % search_str + pci_ids = [id for id in commands.getoutput(cmd).splitlines()] + nic_id = int(re.search('[0-9]+', name).group(0)) + if (len(pci_ids) - 1) < nic_id: + return None + return pci_ids[nic_id] + + def _echo_to_file(self, content, file_path): + """ + Simply echo some content to file. + """ + cmd = "echo %s > %s" % (content, file_path) + status, output = commands.getstatusoutput(cmd) + if status != 0: + return False + return True + + def is_binded_to_stub(self, full_id): + """ + Verify whether the device with full_id is already binded to pci-stub. + """ + base_dir = "/sys/bus/pci" + stub_path = os.path.join(base_dir, "drivers/pci-stub") + if os.path.exists(os.path.join(stub_path, full_id)): + return True + return False + + def request_devs(self, count, pt_file, setup=True): + """ + Implement setup process: unbind the PCI device and then bind it + to pci-stub driver. + + @return: a list of successfully requested devices' PCI IDs. + """ + base_dir = "/sys/bus/pci" + stub_path = os.path.join(base_dir, "drivers/pci-stub") + + self.pci_ids = self.catch_devs(int(count)) + logging.debug("Catch_devs has found pci_ids: %s" % self.pci_ids) + + if setup: + requested_pci_ids = [] + logging.debug("Going to setup devices: %s" % self.pci_ids) + dev_prev_drivers = {} + + # Setup all devices specified for passthrough to guest + for pci_id in self.pci_ids: + full_id = get_full_id(pci_id) + if not full_id: + continue + drv_path = os.path.join(base_dir, "devices/%s/driver" % full_id) + dev_prev_driver= os.path.realpath(os.path.join(drv_path, + os.readlink(drv_path))) + dev_prev_drivers[pci_id] = dev_prev_driver + + # Judge whether the device driver has been binded to stub + if not self.is_binded_to_stub(full_id): + logging.debug("Setting up device: %s" % full_id) + stub_new_id = os.path.join(stub_path, 'new_id') + unbind_dev = os.path.join(drv_path, 'unbind') + stub_bind = os.path.join(stub_path, 'bind') + + vendor_id = get_vendor_id(pci_id) + if not self._echo_to_file(vendor_id, stub_new_id): + continue + if not self._echo_to_file(full_id, unbind_dev): + continue + if not self._echo_to_file(full_id, stub_bind): + continue + if not self.is_binded_to_stub(full_id): + logging.error("Request device failed: %s" % pci_id) + continue + else: + logging.debug("Device [%s] already binded to stub" % pci_id) + requested_pci_ids.append(pci_id) + cPickle.dump(dev_prev_drivers, pt_file) + self.pci_ids = requested_pci_ids + return self.pci_ids + + def catch_vf_devs(self): + """ + Catch all VFs PCI IDs and return as a list. + """ + if not self.sr_iov_setup(): + return [] + + cmd = "lspci | awk '/Virtual Function/ {print $1}'" + return commands.getoutput(cmd).split() + + def catch_pf_devs(self): + """ + Catch all PFs PCI IDs. + """ + pf_ids = [] + for name in self.name_list: + pf_id = self._get_pci_id(name, "Ethernet") + if not pf_id: + continue + pf_ids.append(pf_id) + return pf_ids + + def catch_devs(self, count): + """ + Check out all devices' PCI IDs according to their name. + + @param count: count number of PCI devices needed for pass through + @return: a list of all devices' PCI IDs + """ + if self.type == "nic_vf": + vf_ids = self.catch_vf_devs() + elif self.type == "nic_pf": + vf_ids = self.catch_pf_devs() + elif self.type == "mixed": + vf_ids = self.catch_vf_devs() + vf_ids.extend(self.catch_pf_devs()) + return vf_ids[0:count] diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py index bd7e0fc..90827a2 100755 --- a/client/tests/kvm/kvm_vm.py +++ b/client/tests/kvm/kvm_vm.py @@ -1,5 +1,5 @@ #!/usr/bin/python -import time, socket, os, logging, fcntl, re, commands +import time, socket, os, logging, fcntl, re, commands, cPickle import kvm_utils, kvm_subprocess """ @@ -283,6 +283,29 @@ class VM: elif params.get("uuid"): qemu_cmd += " -uuid %s" % params.get("uuid") + # Check whether pci_passthrough is turned on + if not params.get("pass_through") == "no": + pt_type = params.get("pass_through") + + # Virtual Functions pass through + if params.get("pass_through") == "nic_vf": + pt = kvm_utils.PassThrough(type = pt_type, + max_vfs = params.get("max_vfs")) + # Physical NIC card pass through + elif params.get("pass_through") == "nic_pf": + pt = kvm_utils.PassThrough(type=pt_type, + names=params.get("passthrough_devs")) + elif params.get("pass_through") == "mixed": + pt = kvm_utils.PassThrough(type=pt_type, + max_vfs = params.get("max_vfs"), + names=params.get("passthrough_devs")) + pt_count = params.get("passthrough_count") + pt_db_file = open("/tmp/passthrough_db", "w") + self.pt_pci_ids = pt.request_devs(pt_count, pt_file=pt_db_file, setup=False) + pt_db_file.close() + for pci_id in self.pt_pci_ids: + qemu_cmd += " -pcidevice host=%s" % pci_id + return qemu_cmd @@ -365,6 +388,35 @@ class VM: self.uuid = f.read().strip() f.close() + # Request specified PCI devices for assignment + if not params.get("pass_through") == "no": + pt_type = params.get("pass_through") + + # Virtual Functions pass through + if params.get("pass_through") == "nic_vf": + logging.debug("Going to assign VFs to guest") + pt = kvm_utils.PassThrough(type = pt_type, + max_vfs = params.get("max_vfs")) + # Physical NIC card pass through + elif params.get("pass_through") == "nic_pf": + logging.debug("Going to assign physical NICs to guest") + pt = kvm_utils.PassThrough(type=pt_type, + names=params.get("passthrough_devs")) + elif params.get("pass_through") == "mixed": + logging.debug("Assigned NICs may contain both VFs and PFs") + pt = kvm_utils.PassThrough(type = pt_type, + max_vfs = params.get("max_vfs"), + names = params.get("passthrough_devs")) + pt_count = params.get("passthrough_count") + pt_db_file = open('/tmp/passthrough_db', 'w') + self.pt_pci_ids = pt.request_devs(pt_count, pt_file=pt_db_file, setup=True) + pt_db_file.close() + if not self.pt_pci_ids: + logging.error("Request specified PCI device(s) failed;" + " pt_pci_ids are: %s" % self.pt_pci_ids) + return False + logging.debug("Requested host PCI device(s): %s" % self.pt_pci_ids) + # Make qemu command qemu_command = self.make_qemu_command() @@ -501,11 +553,22 @@ class VM: using a shell command before trying to end the qemu process with a 'quit' or a kill signal. """ + # Release the assigned PCI devices or VFs to host + def release_pt_devs(): + try: + db_file = open('/tmp/passthrough_db', 'r') + pt_db = cPickle.load(db_file) + kvm_utils.release_pci_devs(pt_db) + db_file.close() + except: + return + # Is it already dead? if self.is_dead(): logging.debug("VM is already down") if self.process: self.process.close() + release_pt_devs() return logging.debug("Destroying VM with PID %d..." % self.process.get_pid()) @@ -526,6 +589,7 @@ class VM: return finally: session.close() + release_pt_devs() # Try to destroy with a monitor command logging.debug("Trying to kill VM with monitor command...") @@ -536,6 +600,7 @@ class VM: if kvm_utils.wait_for(self.is_dead, 5, 0.5, 0.5): logging.debug("VM is down") self.process.close() + release_pt_devs() return # If the VM isn't dead yet... @@ -546,6 +611,7 @@ class VM: if kvm_utils.wait_for(self.is_dead, 5, 0.5, 0.5): logging.debug("VM is down") self.process.close() + release_pt_devs() return logging.error("Process %s is a zombie!" % self.process.get_pid())