From patchwork Sun Dec 27 23:55:56 2009 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Lucas Meneghel Rodrigues X-Patchwork-Id: 69913 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter.kernel.org (8.14.3/8.14.2) with ESMTP id nBRNuBL0020789 for ; Sun, 27 Dec 2009 23:56:11 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751281AbZL0X4H (ORCPT ); Sun, 27 Dec 2009 18:56:07 -0500 Received: (majordomo@vger.kernel.org) by vger.kernel.org id S1751202AbZL0X4H (ORCPT ); Sun, 27 Dec 2009 18:56:07 -0500 Received: from mx1.redhat.com ([209.132.183.28]:13710 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751181AbZL0X4F (ORCPT ); Sun, 27 Dec 2009 18:56:05 -0500 Received: from int-mx02.intmail.prod.int.phx2.redhat.com (int-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.12]) by mx1.redhat.com (8.13.8/8.13.8) with ESMTP id nBRNu1Le006829 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=OK); Sun, 27 Dec 2009 18:56:01 -0500 Received: from localhost.localdomain (vpn-8-206.rdu.redhat.com [10.11.8.206]) by int-mx02.intmail.prod.int.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id nBRNtx8C016994; Sun, 27 Dec 2009 18:55:59 -0500 From: Lucas Meneghel Rodrigues To: autotest@test.kernel.org Cc: kvm@vger.kernel.org, yzhou@redhat.com, Lucas Meneghel Rodrigues Subject: [PATCH] KVM test: Add PCI device assignment support Date: Sun, 27 Dec 2009 21:55:56 -0200 Message-Id: <1261958156-14136-1-git-send-email-lmr@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 2.67 on 10.5.11.12 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py index 2bbbe22..59c72a9 100644 --- a/client/tests/kvm/kvm_utils.py +++ b/client/tests/kvm/kvm_utils.py @@ -924,3 +924,284 @@ def create_report(report_dir, results_dir): reporter = os.path.join(report_dir, 'html_report.py') html_file = os.path.join(results_dir, 'results.html') os.system('%s -r %s -f %s -R' % (reporter, results_dir, html_file)) + + +def get_full_pci_id(pci_id): + """ + Get full PCI ID of pci_id. + + @param pci_id: PCI ID of a device. + """ + cmd = "lspci -D | awk '/%s/ {print $1}'" % pci_id + status, full_id = commands.getstatusoutput(cmd) + if status != 0: + return None + return full_id + + +def get_vendor_from_pci_id(pci_id): + """ + Check out the device vendor ID according to pci_id. + + @param pci_id: PCI ID of a device. + """ + cmd = "lspci -n | awk '/%s/ {print $3}'" % pci_id + return re.sub(":", " ", commands.getoutput(cmd)) + + +class PciAssignable(object): + """ + Request PCI assignable devices on host. It will check whether to request + PF (physical Functions) or VF (Virtual Functions). + """ + def __init__(self, type="nic_vf", driver=None, driver_option=None, + names=None, devices_requested=None): + """ + Initialize parameter 'type' which could be: + nic_vf: Virtual Functions + nic_pf: Physical Function (actual hardware) + mixed: Both includes VFs and PFs + + If pass through Physical NIC cards, we need to specify which devices + to be assigned, e.g. 'eth1 eth2'. + + If pass through Virtual Functions, we need to specify how many vfs + are going to be assigned, e.g. passthrough_count = 8 and max_vfs in + config file. + + @param type: PCI device type. + @param driver: Kernel module for the PCI assignable device. + @param driver_option: Module option to specify the maximum number of + VFs (eg 'max_vfs=7') + @param names: Physical NIC cards correspondent network interfaces, + e.g.'eth1 eth2 ...' + """ + self.type = type + self.driver = driver + self.driver_option = driver_option + if names: + self.name_list = names.split() + if devices_requested: + self.devices_requested = int(devices_requested) + + + def _get_pf_pci_id(self, name, search_str): + """ + Get the PF PCI ID according to name. + + @param name: Name of the PCI device. + @param search_str: Search string to be used on lspci. + """ + cmd = "ethtool -i %s | awk '/bus-info/ {print $2}'" % name + s, pci_id = commands.getstatusoutput(cmd) + if not (s or "Cannot get driver information" in pci_id): + return pci_id[5:] + cmd = "lspci | awk '/%s/ {print $1}'" % search_str + pci_ids = [id for id in commands.getoutput(cmd).splitlines()] + nic_id = int(re.search('[0-9]+', name).group(0)) + if (len(pci_ids) - 1) < nic_id: + return None + return pci_ids[nic_id] + + + def _release_dev(self, pci_id): + """ + Release a single PCI device. + + @param pci_id: PCI ID of a given PCI device. + """ + base_dir = "/sys/bus/pci" + full_id = get_full_pci_id(pci_id) + vendor_id = get_vendor_from_pci_id(pci_id) + drv_path = os.path.join(base_dir, "devices/%s/driver" % full_id) + if 'pci-stub' in os.readlink(drv_path): + cmd = "echo '%s' > %s/new_id" % (vendor_id, drv_path) + if os.system(cmd): + return False + + stub_path = os.path.join(base_dir, "drivers/pci-stub") + cmd = "echo '%s' > %s/unbind" % (full_id, stub_path) + if os.system(cmd): + return False + + driver = self.dev_drivers[pci_id] + cmd = "echo '%s' > %s/bind" % (full_id, driver) + if os.system(cmd): + return False + + return True + + + def get_vf_devs(self): + """ + Catch all VFs PCI IDs. + + @return: List with all PCI IDs for the Virtual Functions avaliable + """ + if not self.sr_iov_setup(): + return [] + + cmd = "lspci | awk '/Virtual Function/ {print $1}'" + return commands.getoutput(cmd).split() + + + def get_pf_devs(self): + """ + Catch all PFs PCI IDs. + + @return: List with all PCI IDs for the physical hardware requested + """ + pf_ids = [] + for name in self.name_list: + pf_id = self._get_pf_pci_id(name, "Ethernet") + if not pf_id: + continue + pf_ids.append(pf_id) + return pf_ids + + + def get_devs(self, count): + """ + Check out all devices' PCI IDs according to their name. + + @param count: count number of PCI devices needed for pass through + @return: a list of all devices' PCI IDs + """ + if self.type == "nic_vf": + vf_ids = self.get_vf_devs() + elif self.type == "nic_pf": + vf_ids = self.get_pf_devs() + elif self.type == "mixed": + vf_ids = self.get_vf_devs() + vf_ids.extend(self.get_pf_devs()) + return vf_ids[0:count] + + + def get_vfs_count(self): + """ + Get VFs count number according to lspci. + """ + cmd = "lspci | grep 'Virtual Function' | wc -l" + # For each VF we'll see 2 prints of 'Virtual Function', so let's + # divide the result per 2 + return int(commands.getoutput(cmd)) / 2 + + + def check_vfs_count(self): + """ + Check VFs count number according to the parameter driver_options. + """ + return (self.get_vfs_count == self.devices_requested) + + + def is_binded_to_stub(self, full_id): + """ + Verify whether the device with full_id is already binded to pci-stub. + + @param full_id: Full ID for the given PCI device + """ + base_dir = "/sys/bus/pci" + stub_path = os.path.join(base_dir, "drivers/pci-stub") + if os.path.exists(os.path.join(stub_path, full_id)): + return True + return False + + + def sr_iov_setup(self): + """ + Ensure the PCI device is working in sr_iov mode. + + Check if the PCI hardware device drive is loaded with the appropriate, + parameters (number of VFs), and if it's not, perform setup. + + @return: True, if the setup was completed successfuly, False otherwise. + """ + re_probe = False + s, o = commands.getstatusoutput('lsmod | grep %s' % self.driver) + if s: + re_probe = True + elif not self.check_vfs_count(): + os.system("modprobe -r %s" % self.driver) + re_probe = True + + # Re-probe driver with proper number of VFs + if re_probe: + cmd = "modprobe %s %s" % (self.driver, self.driver_option) + s, o = commands.getstatusoutput(cmd) + if s: + return False + if not self.check_vfs_count(): + return False + return True + + + def request_devs(self): + """ + Implement setup process: unbind the PCI device and then bind it + to the pci-stub driver. + + @return: a list of successfully requested devices' PCI IDs. + """ + base_dir = "/sys/bus/pci" + stub_path = os.path.join(base_dir, "drivers/pci-stub") + + self.pci_ids = self.get_devs(self.devices_requested) + logging.debug("The following pci_ids were found: %s" % self.pci_ids) + requested_pci_ids = [] + self.dev_drivers = {} + + # Setup all devices specified for assignment to guest + for pci_id in self.pci_ids: + full_id = get_full_pci_id(pci_id) + if not full_id: + continue + drv_path = os.path.join(base_dir, "devices/%s/driver" % full_id) + dev_prev_driver= os.path.realpath(os.path.join(drv_path, + os.readlink(drv_path))) + self.dev_drivers[pci_id] = dev_prev_driver + + # Judge whether the device driver has been binded to stub + if not self.is_binded_to_stub(full_id): + logging.debug("Binding device %s to stub" % full_id) + vendor_id = get_vendor_from_pci_id(pci_id) + stub_new_id = os.path.join(stub_path, 'new_id') + unbind_dev = os.path.join(drv_path, 'unbind') + stub_bind = os.path.join(stub_path, 'bind') + + info_write_to_files = [(vendor_id, stub_new_id), + (full_id, unbind_dev), + (full_id, stub_bind)] + + for content, file in info_write_to_files: + try: + utils.open_write_close(content, file) + except IOError: + logging.debug("Failed to write %s to file %s" % + (content, file)) + continue + + if not self.is_binded_to_stub(full_id): + logging.error("Binding device %s to stub failed" % + pci_id) + continue + else: + logging.debug("Device %s already binded to stub" % pci_id) + requested_pci_ids.append(pci_id) + self.pci_ids = requested_pci_ids + return self.pci_ids + + + def release_devs(self): + """ + Release all PCI devices currently assigned to VMs back to the + virtualization host. + """ + try: + for pci_id in self.dev_drivers: + if not self._release_dev(pci_id): + logging.error("Failed to release device %s to host" % + pci_id) + else: + logging.info("Released device %s successfully" % pci_id) + except: + return diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py index cc314d4..a86c124 100755 --- a/client/tests/kvm/kvm_vm.py +++ b/client/tests/kvm/kvm_vm.py @@ -304,6 +304,12 @@ class VM: elif params.get("uuid"): qemu_cmd += " -uuid %s" % params.get("uuid") + # If the PCI assignment step went OK, add each one of the PCI assigned + # devices to the qemu command line. + if self.pci_assignable: + for pci_id in self.pa_pci_ids: + qemu_cmd += " -pcidevice host=%s" % pci_id + return qemu_cmd @@ -392,6 +398,50 @@ class VM: self.uuid = f.read().strip() f.close() + if not params.get("pci_assignable") == "no": + pa_type = params.get("pci_assignable") + pa_devices_requested = params.get("devices_requested") + + # Virtual Functions (VF) assignable devices + if pa_type == "vf": + pa_driver = params.get("driver") + pa_driver_option = params.get("driver_option") + self.pci_assignable = kvm_utils.PciAssignable(type=pa_type, + driver=pa_driver, + driver_option=pa_driver_option, + devices_requested=pa_devices_requested) + # Physical NIC (PF) assignable devices + elif pa_type == "pf": + pa_device_names = params.get("device_names") + self.pci_assignable = kvm_utils.PciAssignable(type=pa_type, + names=pa_device_names, + devices_requested=pa_devices_requested) + # Working with both VF and PF + elif pa_type == "mixed": + pa_device_names = params.get("device_names") + pa_driver = params.get("driver") + pa_driver_option = params.get("driver_option") + self.pci_assignable = kvm_utils.PciAssignable(type=pa_type, + driver=pa_driver, + driver_option=pa_driver_option, + names=pa_device_names, + devices_requested=pa_devices_requested) + + self.pa_pci_ids = self.pci_assignable.request_devs() + + if self.pa_pci_ids: + logging.debug("Successfuly assigned devices: %s" % + self.pa_pci_ids) + else: + logging.error("No PCI assignable devices were assigned " + "and 'pci_assignable' is defined to %s " + "on your config file. Aborting VM creation." % + pa_type) + return False + + else: + self.pci_assignable = None + # Make qemu command qemu_command = self.make_qemu_command() @@ -537,6 +587,8 @@ class VM: # Is it already dead? if self.is_dead(): logging.debug("VM is already down") + if self.pci_assignable: + self.pci_assignable.release_devs() return logging.debug("Destroying VM with PID %d..." % @@ -557,6 +609,9 @@ class VM: return finally: session.close() + if self.pci_assignable: + self.pci_assignable.release_devs() + # Try to destroy with a monitor command logging.debug("Trying to kill VM with monitor command...") @@ -566,6 +621,8 @@ class VM: # Wait for the VM to be really dead if kvm_utils.wait_for(self.is_dead, 5, 0.5, 0.5): logging.debug("VM is down") + if self.pci_assignable: + self.pci_assignable.release_devs() return # If the VM isn't dead yet... @@ -575,6 +632,8 @@ class VM: # Wait for the VM to be really dead if kvm_utils.wait_for(self.is_dead, 5, 0.5, 0.5): logging.debug("VM is down") + if self.pci_assignable: + self.pci_assignable.release_devs() return logging.error("Process %s is a zombie!" % self.process.get_pid()) diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample index a403399..b7ee2e1 100644 --- a/client/tests/kvm/tests_base.cfg.sample +++ b/client/tests/kvm/tests_base.cfg.sample @@ -884,3 +884,23 @@ variants: pre_command = "/usr/bin/python scripts/hugepage.py /mnt/kvm_hugepage" extra_params += " -mem-path /mnt/kvm_hugepage" + +variants: + - @no_pci_assignable: + pci_assignable = no + - pf_assignable: + pci_assignable = pf + device_names = eth1 + - vf_assignable: + pci_assignable = vf + # Driver (kernel module) that supports SR-IOV hardware. + # As of today (30-11-2009), we have 2 drivers for this type of hardware: + # Intel® 82576 Gigabit Ethernet Controller - igb + # Neterion® X3100™ - vxge + driver = igb + # Driver option to specify the maximum number of virtual functions + # (on vxge the option is , for example, is max_config_dev) + # the default below is for the igb driver + driver_option = "max_vfs=7" + # Number of devices that are going to be requested. + devices_requested = 7