| Message ID | 1477495586-5508-2-git-send-email-felipe@nutanix.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On 26/10/2016 17:26, Felipe Franciosi wrote: > This commit introduces a vhost-user device for SCSI. This is based > on the existing vhost-scsi implementation, but done over vhost-user > instead. It also uses a chardev to connect to the backend. Unlike > vhost-scsi (today), VMs using vhost-user-scsi can be live migrated. > > To use it, one must configure Qemu with --enable-vhost-user-scsi and > start Qemu with a command line equivalent to: > > qemu-system-x86_64 \ > -chardev socket,id=vus0,path=/tmp/vus.sock \ > -device vhost-user-scsi-pci,chardev=vus0,bus=pci.0,addr=0x4 > > A separate commit presents a sample application linked with libiscsi to > provide a backend for vhost-user-scsi. Hi, most of the code you copy can be kept in one place only, by using the vhost_ops struct. Please make a hierarchy virtio-scsi-common vhost-scsi-common vhost-scsi (adds vhostfd+wwpn) vhost-user-scsi (adds chardev) with abstract methods in vhost-scsi-common to abstract e.g. opening /dev/scsi and setting/clearing the endpoint, which are only needed by vhost-scsi's realize method. This should avoid the duplication. Thanks! Paolo > Signed-off-by: Felipe Franciosi <felipe@nutanix.com> > --- > configure | 10 ++ > hw/scsi/Makefile.objs | 1 + > hw/scsi/vhost-user-scsi.c | 299 ++++++++++++++++++++++++++++++++++++ > hw/virtio/virtio-pci.c | 58 +++++++ > hw/virtio/virtio-pci.h | 15 ++ > include/hw/virtio/vhost-user-scsi.h | 45 ++++++ > include/hw/virtio/virtio-scsi.h | 5 + > 7 files changed, 433 insertions(+) > create mode 100644 hw/scsi/vhost-user-scsi.c > create mode 100644 include/hw/virtio/vhost-user-scsi.h > > diff --git a/configure b/configure > index d3dafcb..0574ff2 100755 > --- a/configure > +++ b/configure > @@ -228,6 +228,7 @@ xfs="" > > vhost_net="no" > vhost_scsi="no" > +vhost_user_scsi="no" > vhost_vsock="no" > kvm="no" > rdma="" > @@ -677,6 +678,7 @@ Haiku) > kvm="yes" > vhost_net="yes" > vhost_scsi="yes" > + vhost_user_scsi="yes" > vhost_vsock="yes" > QEMU_INCLUDES="-I\$(SRC_PATH)/linux-headers -I$(pwd)/linux-headers $QEMU_INCLUDES" > ;; > @@ -1019,6 +1021,10 @@ for opt do > ;; > --enable-vhost-scsi) vhost_scsi="yes" > ;; > + --disable-vhost-user-scsi) vhost_user_scsi="no" > + ;; > + --enable-vhost-user-scsi) vhost_user_scsi="yes" > + ;; > --disable-vhost-vsock) vhost_vsock="no" > ;; > --enable-vhost-vsock) vhost_vsock="yes" > @@ -4951,6 +4957,7 @@ echo "posix_madvise $posix_madvise" > echo "libcap-ng support $cap_ng" > echo "vhost-net support $vhost_net" > echo "vhost-scsi support $vhost_scsi" > +echo "vhost-user-scsi support $vhost_user_scsi" > echo "vhost-vsock support $vhost_vsock" > echo "Trace backends $trace_backends" > if have_backend "simple"; then > @@ -5336,6 +5343,9 @@ fi > if test "$vhost_scsi" = "yes" ; then > echo "CONFIG_VHOST_SCSI=y" >> $config_host_mak > fi > +if test "$vhost_user_scsi" = "yes" ; then > + echo "CONFIG_VHOST_USER_SCSI=y" >> $config_host_mak > +fi > if test "$vhost_net" = "yes" ; then > echo "CONFIG_VHOST_NET_USED=y" >> $config_host_mak > fi > diff --git a/hw/scsi/Makefile.objs b/hw/scsi/Makefile.objs > index 5a2248b..3338aad 100644 > --- a/hw/scsi/Makefile.objs > +++ b/hw/scsi/Makefile.objs > @@ -11,4 +11,5 @@ obj-$(CONFIG_PSERIES) += spapr_vscsi.o > ifeq ($(CONFIG_VIRTIO),y) > obj-y += virtio-scsi.o virtio-scsi-dataplane.o > obj-$(CONFIG_VHOST_SCSI) += vhost-scsi.o > +obj-$(CONFIG_VHOST_USER_SCSI) += vhost-user-scsi.o > endif > diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c > new file mode 100644 > index 0000000..a7b9839 > --- /dev/null > +++ b/hw/scsi/vhost-user-scsi.c > @@ -0,0 +1,299 @@ > +/* > + * vhost-user-scsi host device > + * > + * Copyright (c) 2016 Nutanix Inc. All rights reserved. > + * > + * Author: > + * Felipe Franciosi <felipe@nutanix.com> > + * > + * This work is largely based on the "vhost-scsi" implementation by: > + * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com> > + * Nicholas Bellinger <nab@risingtidesystems.com> > + * > + * This work is licensed under the terms of the GNU LGPL, version 2 or later. > + * See the COPYING.LIB file in the top-level directory. > + * > + */ > + > +#include "qemu/osdep.h" > +#include "migration/vmstate.h" > +#include "qapi/error.h" > +#include "qemu/error-report.h" > +#include "qemu/typedefs.h" > +#include "qom/object.h" > +#include "hw/fw-path-provider.h" > +#include "hw/qdev-core.h" > +#include "hw/virtio/vhost.h" > +#include "hw/virtio/vhost-backend.h" > +#include "hw/virtio/vhost-user-scsi.h" > +#include "hw/virtio/virtio.h" > +#include "hw/virtio/virtio-access.h" > +#include "hw/virtio/virtio-bus.h" > +#include "sysemu/char.h" > + > +/* Features supported by the host application */ > +static const int user_feature_bits[] = { > + VIRTIO_F_NOTIFY_ON_EMPTY, > + VIRTIO_RING_F_INDIRECT_DESC, > + VIRTIO_RING_F_EVENT_IDX, > + VIRTIO_SCSI_F_HOTPLUG, > + VHOST_INVALID_FEATURE_BIT > +}; > + > +static int vhost_user_scsi_start(VHostUserSCSI *s) > +{ > + int ret, i; > + VirtIODevice *vdev = VIRTIO_DEVICE(s); > + BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); > + VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); > + > + if (!k->set_guest_notifiers) { > + error_report("binding does not support guest notifiers"); > + return -ENOSYS; > + } > + > + ret = vhost_dev_enable_notifiers(&s->dev, vdev); > + if (ret < 0) { > + return ret; > + } > + > + s->dev.acked_features = vdev->guest_features; > + ret = vhost_dev_start(&s->dev, vdev); > + if (ret < 0) { > + error_report("Error starting vhost-user device"); > + goto err_notifiers; > + } > + > + ret = k->set_guest_notifiers(qbus->parent, s->dev.nvqs, true); > + if (ret < 0) { > + error_report("Error binding guest notifier"); > + goto err_vhost_stop; > + } > + > + /* guest_notifier_mask/pending not used yet, so just unmask > + * everything here. virtio-pci will do the right thing by > + * enabling/disabling irqfd. > + */ > + for (i = 0; i < s->dev.nvqs; i++) { > + vhost_virtqueue_mask(&s->dev, vdev, s->dev.vq_index + i, false); > + } > + > + return ret; > + > +err_vhost_stop: > + vhost_dev_stop(&s->dev, vdev); > +err_notifiers: > + vhost_dev_disable_notifiers(&s->dev, vdev); > + return ret; > +} > + > +static void vhost_user_scsi_stop(VHostUserSCSI *s) > +{ > + VirtIODevice *vdev = VIRTIO_DEVICE(s); > + BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); > + VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); > + int ret = 0; > + > + vhost_dev_stop(&s->dev, vdev); > + vhost_dev_disable_notifiers(&s->dev, vdev); > + > + if (k->set_guest_notifiers) { > + ret = k->set_guest_notifiers(qbus->parent, s->dev.nvqs, false); > + if (ret < 0) { > + error_report("vhost guest notifier cleanup failed: %d", ret); > + } > + } > + assert(ret >= 0); > +} > + > +static uint64_t vhost_user_scsi_get_features(VirtIODevice *vdev, > + uint64_t features, > + Error **errp) > +{ > + VHostUserSCSI *s = VHOST_USER_SCSI(vdev); > + > + return vhost_get_features(&s->dev, user_feature_bits, features); > +} > + > +static void vhost_user_scsi_set_config(VirtIODevice *vdev, > + const uint8_t *config) > +{ > + VirtIOSCSIConfig *scsiconf = (VirtIOSCSIConfig *)config; > + VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(vdev); > + > + if ((uint32_t)virtio_ldl_p(vdev, &scsiconf->sense_size) != vs->sense_size || > + (uint32_t)virtio_ldl_p(vdev, &scsiconf->cdb_size) != vs->cdb_size) { > + error_report("vhost-user-scsi doesn't allow sense or CDB sizes change"); > + exit(1); > + } > +} > + > +static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status) > +{ > + VHostUserSCSI *s = (VHostUserSCSI *)vdev; > + bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running; > + > + if (s->dev.started == start) { > + return; > + } > + > + if (start) { > + int ret; > + > + ret = vhost_user_scsi_start(s); > + if (ret < 0) { > + error_report("unable to start vhost-user-scsi: %s", strerror(-ret)); > + exit(1); > + } > + } else { > + vhost_user_scsi_stop(s); > + } > +} > + > +static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq) > +{ > +} > + > +static void vhost_user_scsi_save(QEMUFile *f, void *opaque) > +{ > + VirtIODevice *vdev = VIRTIO_DEVICE(opaque); > + virtio_save(vdev, f); > +} > + > +static int vhost_user_scsi_load(QEMUFile *f, void *opaque, int version_id) > +{ > + VirtIODevice *vdev = VIRTIO_DEVICE(opaque); > + return virtio_load(vdev, f, version_id); > +} > + > +static void vhost_user_scsi_realize(DeviceState *dev, Error **errp) > +{ > + VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev); > + VHostUserSCSI *s = VHOST_USER_SCSI(dev); > + static int vhost_user_scsi_id; > + Error *err = NULL; > + int ret; > + > + if (!vs->conf.chardev.chr) { > + error_setg(errp, "vhost-user-scsi: missing chardev"); > + return; > + } > + > + virtio_scsi_common_realize(dev, &err, vhost_dummy_handle_output, > + vhost_dummy_handle_output, > + vhost_dummy_handle_output); > + if (err != NULL) { > + error_propagate(errp, err); > + return; > + } > + > + s->dev.nvqs = VHOST_USER_SCSI_VQ_NUM_FIXED + vs->conf.num_queues; > + s->dev.vqs = g_new(struct vhost_virtqueue, s->dev.nvqs); > + s->dev.vq_index = 0; > + s->dev.backend_features = 0; > + > + ret = vhost_dev_init(&s->dev, (void *)&vs->conf.chardev, > + VHOST_BACKEND_TYPE_USER, 0); > + if (ret < 0) { > + error_setg(errp, "vhost-user-scsi: vhost initialization failed: %s", > + strerror(-ret)); > + return; > + } > + > + /* Channel and lun both are 0 for bootable vhost-user-scsi disk */ > + s->channel = 0; > + s->lun = 0; > + s->target = vs->conf.boot_tpgt; > + > + register_savevm(dev, "vhost-user-scsi", vhost_user_scsi_id++, 1, > + vhost_user_scsi_save, vhost_user_scsi_load, s); > +} > + > +static void vhost_user_scsi_unrealize(DeviceState *dev, Error **errp) > +{ > + VirtIODevice *vdev = VIRTIO_DEVICE(dev); > + VHostUserSCSI *s = VHOST_USER_SCSI(dev); > + > + /* This will stop the vhost backend. */ > + vhost_user_scsi_set_status(vdev, 0); > + > + vhost_dev_cleanup(&s->dev); > + g_free(s->dev.vqs); > + > + virtio_scsi_common_unrealize(dev, errp); > +} > + > +/* > + * Implementation of an interface to adjust firmware path > + * for the bootindex property handling. > + */ > +static char *vhost_user_scsi_get_fw_dev_path(FWPathProvider *p, BusState *bus, > + DeviceState *dev) > +{ > + VHostUserSCSI *s = VHOST_USER_SCSI(dev); > + /* format: /channel@<channel>/vhost-user-scsi@<target>,<lun> */ > + return g_strdup_printf("/channel@%x/%s@%x,%x", s->channel, > + qdev_fw_name(dev), s->target, s->lun); > +} > + > +static Property vhost_user_scsi_properties[] = { > + DEFINE_PROP_CHR("chardev", VHostUserSCSI, > + parent_obj.conf.chardev), > + DEFINE_PROP_UINT32("boot_tpgt", VHostUserSCSI, > + parent_obj.conf.boot_tpgt, 0), > + DEFINE_PROP_UINT32("num_queues", VHostUserSCSI, > + parent_obj.conf.num_queues, 1), > + DEFINE_PROP_UINT32("max_sectors", VHostUserSCSI, > + parent_obj.conf.max_sectors, 0xFFFF), > + DEFINE_PROP_UINT32("cmd_per_lun", VHostUserSCSI, > + parent_obj.conf.cmd_per_lun, 128), > + DEFINE_PROP_END_OF_LIST(), > +}; > + > +static void vhost_user_scsi_class_init(ObjectClass *klass, void *data) > +{ > + DeviceClass *dc = DEVICE_CLASS(klass); > + VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); > + FWPathProviderClass *fwc = FW_PATH_PROVIDER_CLASS(klass); > + > + dc->props = vhost_user_scsi_properties; > + set_bit(DEVICE_CATEGORY_STORAGE, dc->categories); > + vdc->realize = vhost_user_scsi_realize; > + vdc->unrealize = vhost_user_scsi_unrealize; > + vdc->get_features = vhost_user_scsi_get_features; > + vdc->set_config = vhost_user_scsi_set_config; > + vdc->set_status = vhost_user_scsi_set_status; > + fwc->get_dev_path = vhost_user_scsi_get_fw_dev_path; > +} > + > +static void vhost_user_scsi_instance_init(Object *obj) > +{ > + VHostUserSCSI *dev = VHOST_USER_SCSI(obj); > + VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(obj); > + > + // Add the bootindex property for this object > + device_add_bootindex_property(obj, &dev->bootindex, "bootindex", NULL, > + DEVICE(dev), NULL); > + > + // Set boot index according the the device config > + object_property_set_int(obj, vs->conf.bootindex, "bootindex", NULL); > +} > + > +static const TypeInfo vhost_user_scsi_info = { > + .name = TYPE_VHOST_USER_SCSI, > + .parent = TYPE_VIRTIO_SCSI_COMMON, > + .instance_size = sizeof(VHostUserSCSI), > + .class_init = vhost_user_scsi_class_init, > + .instance_init = vhost_user_scsi_instance_init, > + .interfaces = (InterfaceInfo[]) { > + { TYPE_FW_PATH_PROVIDER }, > + { } > + }, > +}; > + > +static void virtio_register_types(void) > +{ > + type_register_static(&vhost_user_scsi_info); > +} > + > +type_init(virtio_register_types) > diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c > index 06831de..b996d37 100644 > --- a/hw/virtio/virtio-pci.c > +++ b/hw/virtio/virtio-pci.c > @@ -2098,6 +2098,61 @@ static const TypeInfo vhost_scsi_pci_info = { > }; > #endif > > +/* vhost-user-scsi-pci */ > +#ifdef CONFIG_VHOST_USER_SCSI > +static Property vhost_user_scsi_pci_properties[] = { > + DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, > + DEV_NVECTORS_UNSPECIFIED), > + DEFINE_PROP_END_OF_LIST(), > +}; > + > +static void vhost_user_scsi_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp) > +{ > + VHostUserSCSIPCI *dev = VHOST_USER_SCSI_PCI(vpci_dev); > + DeviceState *vdev = DEVICE(&dev->vdev); > + VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(vdev); > + > + if (vpci_dev->nvectors == DEV_NVECTORS_UNSPECIFIED) { > + vpci_dev->nvectors = vs->conf.num_queues + 3; > + } > + > + qdev_set_parent_bus(vdev, BUS(&vpci_dev->bus)); > + object_property_set_bool(OBJECT(vdev), true, "realized", errp); > +} > + > +static void vhost_user_scsi_pci_class_init(ObjectClass *klass, void *data) > +{ > + DeviceClass *dc = DEVICE_CLASS(klass); > + VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass); > + PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass); > + k->realize = vhost_user_scsi_pci_realize; > + set_bit(DEVICE_CATEGORY_STORAGE, dc->categories); > + dc->props = vhost_user_scsi_pci_properties; > + pcidev_k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET; > + pcidev_k->device_id = PCI_DEVICE_ID_VIRTIO_SCSI; > + pcidev_k->revision = 0x00; > + pcidev_k->class_id = PCI_CLASS_STORAGE_SCSI; > +} > + > +static void vhost_user_scsi_pci_instance_init(Object *obj) > +{ > + VHostUserSCSIPCI *dev = VHOST_USER_SCSI_PCI(obj); > + > + virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev), > + TYPE_VHOST_USER_SCSI); > + object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev), > + "bootindex", &error_abort); > +} > + > +static const TypeInfo vhost_user_scsi_pci_info = { > + .name = TYPE_VHOST_USER_SCSI_PCI, > + .parent = TYPE_VIRTIO_PCI, > + .instance_size = sizeof(VHostUserSCSIPCI), > + .instance_init = vhost_user_scsi_pci_instance_init, > + .class_init = vhost_user_scsi_pci_class_init, > +}; > +#endif /* CONFIG_VHOST_USER_SCSI */ > + > /* vhost-vsock-pci */ > > #ifdef CONFIG_VHOST_VSOCK > @@ -2577,6 +2632,9 @@ static void virtio_pci_register_types(void) > #ifdef CONFIG_VHOST_SCSI > type_register_static(&vhost_scsi_pci_info); > #endif > +#ifdef CONFIG_VHOST_USER_SCSI > + type_register_static(&vhost_user_scsi_pci_info); > +#endif > #ifdef CONFIG_VHOST_VSOCK > type_register_static(&vhost_vsock_pci_info); > #endif > diff --git a/hw/virtio/virtio-pci.h b/hw/virtio/virtio-pci.h > index b4edea6..404b2c4 100644 > --- a/hw/virtio/virtio-pci.h > +++ b/hw/virtio/virtio-pci.h > @@ -31,6 +31,9 @@ > #ifdef CONFIG_VHOST_SCSI > #include "hw/virtio/vhost-scsi.h" > #endif > +#ifdef CONFIG_VHOST_USER_SCSI > +#include "hw/virtio/vhost-user-scsi.h" > +#endif > #ifdef CONFIG_VHOST_VSOCK > #include "hw/virtio/vhost-vsock.h" > #endif > @@ -42,6 +45,7 @@ typedef struct VirtIOBalloonPCI VirtIOBalloonPCI; > typedef struct VirtIOSerialPCI VirtIOSerialPCI; > typedef struct VirtIONetPCI VirtIONetPCI; > typedef struct VHostSCSIPCI VHostSCSIPCI; > +typedef struct VHostUserSCSIPCI VHostUserSCSIPCI; > typedef struct VirtIORngPCI VirtIORngPCI; > typedef struct VirtIOInputPCI VirtIOInputPCI; > typedef struct VirtIOInputHIDPCI VirtIOInputHIDPCI; > @@ -212,6 +216,17 @@ struct VHostSCSIPCI { > }; > #endif > > +#ifdef CONFIG_VHOST_USER_SCSI > +#define TYPE_VHOST_USER_SCSI_PCI "vhost-user-scsi-pci" > +#define VHOST_USER_SCSI_PCI(obj) \ > + OBJECT_CHECK(VHostUserSCSIPCI, (obj), TYPE_VHOST_USER_SCSI_PCI) > + > +struct VHostUserSCSIPCI { > + VirtIOPCIProxy parent_obj; > + VHostUserSCSI vdev; > +}; > +#endif > + > /* > * virtio-blk-pci: This extends VirtioPCIProxy. > */ > diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h > new file mode 100644 > index 0000000..4a16181 > --- /dev/null > +++ b/include/hw/virtio/vhost-user-scsi.h > @@ -0,0 +1,45 @@ > +/* > + * vhost-user-scsi host device > + * > + * Copyright (c) 2016 Nutanix Inc. All rights reserved. > + * > + * Author: > + * Felipe Franciosi <felipe@nutanix.com> > + * > + * This file is largely based on "vhost-scsi.h" by: > + * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com> > + * > + * This work is licensed under the terms of the GNU LGPL, version 2 or later. > + * See the COPYING.LIB file in the top-level directory. > + * > + */ > + > +#ifndef VHOST_USER_SCSI_H > +#define VHOST_USER_SCSI_H > + > +#include "qemu-common.h" > +#include "hw/qdev.h" > +#include "hw/virtio/virtio-scsi.h" > +#include "hw/virtio/vhost.h" > + > +enum vhost_user_scsi_vq_list { > + VHOST_USER_SCSI_VQ_CONTROL = 0, > + VHOST_USER_SCSI_VQ_EVENT = 1, > + VHOST_USER_SCSI_VQ_NUM_FIXED = 2, > +}; > + > +#define TYPE_VHOST_USER_SCSI "vhost-user-scsi" > +#define VHOST_USER_SCSI(obj) \ > + OBJECT_CHECK(VHostUserSCSI, (obj), TYPE_VHOST_USER_SCSI) > + > +typedef struct VHostUserSCSI { > + VirtIOSCSICommon parent_obj; > + > + struct vhost_dev dev; > + int32_t bootindex; > + int channel; > + int target; > + int lun; > +} VHostUserSCSI; > + > +#endif /* VHOST_USER_SCSI_H */ > diff --git a/include/hw/virtio/virtio-scsi.h b/include/hw/virtio/virtio-scsi.h > index a1e0cfb..4b97386 100644 > --- a/include/hw/virtio/virtio-scsi.h > +++ b/include/hw/virtio/virtio-scsi.h > @@ -21,6 +21,7 @@ > #include "hw/virtio/virtio.h" > #include "hw/pci/pci.h" > #include "hw/scsi/scsi.h" > +#include "sysemu/char.h" > #include "sysemu/iothread.h" > > #define TYPE_VIRTIO_SCSI_COMMON "virtio-scsi-common" > @@ -53,6 +54,10 @@ struct VirtIOSCSIConf { > char *wwpn; > uint32_t boot_tpgt; > IOThread *iothread; > +#ifdef CONFIG_VHOST_USER_SCSI > + CharBackend chardev; > + int32_t bootindex; > +#endif > }; > > struct VirtIOSCSI; >
diff --git a/configure b/configure index d3dafcb..0574ff2 100755 --- a/configure +++ b/configure @@ -228,6 +228,7 @@ xfs="" vhost_net="no" vhost_scsi="no" +vhost_user_scsi="no" vhost_vsock="no" kvm="no" rdma="" @@ -677,6 +678,7 @@ Haiku) kvm="yes" vhost_net="yes" vhost_scsi="yes" + vhost_user_scsi="yes" vhost_vsock="yes" QEMU_INCLUDES="-I\$(SRC_PATH)/linux-headers -I$(pwd)/linux-headers $QEMU_INCLUDES" ;; @@ -1019,6 +1021,10 @@ for opt do ;; --enable-vhost-scsi) vhost_scsi="yes" ;; + --disable-vhost-user-scsi) vhost_user_scsi="no" + ;; + --enable-vhost-user-scsi) vhost_user_scsi="yes" + ;; --disable-vhost-vsock) vhost_vsock="no" ;; --enable-vhost-vsock) vhost_vsock="yes" @@ -4951,6 +4957,7 @@ echo "posix_madvise $posix_madvise" echo "libcap-ng support $cap_ng" echo "vhost-net support $vhost_net" echo "vhost-scsi support $vhost_scsi" +echo "vhost-user-scsi support $vhost_user_scsi" echo "vhost-vsock support $vhost_vsock" echo "Trace backends $trace_backends" if have_backend "simple"; then @@ -5336,6 +5343,9 @@ fi if test "$vhost_scsi" = "yes" ; then echo "CONFIG_VHOST_SCSI=y" >> $config_host_mak fi +if test "$vhost_user_scsi" = "yes" ; then + echo "CONFIG_VHOST_USER_SCSI=y" >> $config_host_mak +fi if test "$vhost_net" = "yes" ; then echo "CONFIG_VHOST_NET_USED=y" >> $config_host_mak fi diff --git a/hw/scsi/Makefile.objs b/hw/scsi/Makefile.objs index 5a2248b..3338aad 100644 --- a/hw/scsi/Makefile.objs +++ b/hw/scsi/Makefile.objs @@ -11,4 +11,5 @@ obj-$(CONFIG_PSERIES) += spapr_vscsi.o ifeq ($(CONFIG_VIRTIO),y) obj-y += virtio-scsi.o virtio-scsi-dataplane.o obj-$(CONFIG_VHOST_SCSI) += vhost-scsi.o +obj-$(CONFIG_VHOST_USER_SCSI) += vhost-user-scsi.o endif diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c new file mode 100644 index 0000000..a7b9839 --- /dev/null +++ b/hw/scsi/vhost-user-scsi.c @@ -0,0 +1,299 @@ +/* + * vhost-user-scsi host device + * + * Copyright (c) 2016 Nutanix Inc. All rights reserved. + * + * Author: + * Felipe Franciosi <felipe@nutanix.com> + * + * This work is largely based on the "vhost-scsi" implementation by: + * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com> + * Nicholas Bellinger <nab@risingtidesystems.com> + * + * This work is licensed under the terms of the GNU LGPL, version 2 or later. + * See the COPYING.LIB file in the top-level directory. + * + */ + +#include "qemu/osdep.h" +#include "migration/vmstate.h" +#include "qapi/error.h" +#include "qemu/error-report.h" +#include "qemu/typedefs.h" +#include "qom/object.h" +#include "hw/fw-path-provider.h" +#include "hw/qdev-core.h" +#include "hw/virtio/vhost.h" +#include "hw/virtio/vhost-backend.h" +#include "hw/virtio/vhost-user-scsi.h" +#include "hw/virtio/virtio.h" +#include "hw/virtio/virtio-access.h" +#include "hw/virtio/virtio-bus.h" +#include "sysemu/char.h" + +/* Features supported by the host application */ +static const int user_feature_bits[] = { + VIRTIO_F_NOTIFY_ON_EMPTY, + VIRTIO_RING_F_INDIRECT_DESC, + VIRTIO_RING_F_EVENT_IDX, + VIRTIO_SCSI_F_HOTPLUG, + VHOST_INVALID_FEATURE_BIT +}; + +static int vhost_user_scsi_start(VHostUserSCSI *s) +{ + int ret, i; + VirtIODevice *vdev = VIRTIO_DEVICE(s); + BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); + VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); + + if (!k->set_guest_notifiers) { + error_report("binding does not support guest notifiers"); + return -ENOSYS; + } + + ret = vhost_dev_enable_notifiers(&s->dev, vdev); + if (ret < 0) { + return ret; + } + + s->dev.acked_features = vdev->guest_features; + ret = vhost_dev_start(&s->dev, vdev); + if (ret < 0) { + error_report("Error starting vhost-user device"); + goto err_notifiers; + } + + ret = k->set_guest_notifiers(qbus->parent, s->dev.nvqs, true); + if (ret < 0) { + error_report("Error binding guest notifier"); + goto err_vhost_stop; + } + + /* guest_notifier_mask/pending not used yet, so just unmask + * everything here. virtio-pci will do the right thing by + * enabling/disabling irqfd. + */ + for (i = 0; i < s->dev.nvqs; i++) { + vhost_virtqueue_mask(&s->dev, vdev, s->dev.vq_index + i, false); + } + + return ret; + +err_vhost_stop: + vhost_dev_stop(&s->dev, vdev); +err_notifiers: + vhost_dev_disable_notifiers(&s->dev, vdev); + return ret; +} + +static void vhost_user_scsi_stop(VHostUserSCSI *s) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(s); + BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); + VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); + int ret = 0; + + vhost_dev_stop(&s->dev, vdev); + vhost_dev_disable_notifiers(&s->dev, vdev); + + if (k->set_guest_notifiers) { + ret = k->set_guest_notifiers(qbus->parent, s->dev.nvqs, false); + if (ret < 0) { + error_report("vhost guest notifier cleanup failed: %d", ret); + } + } + assert(ret >= 0); +} + +static uint64_t vhost_user_scsi_get_features(VirtIODevice *vdev, + uint64_t features, + Error **errp) +{ + VHostUserSCSI *s = VHOST_USER_SCSI(vdev); + + return vhost_get_features(&s->dev, user_feature_bits, features); +} + +static void vhost_user_scsi_set_config(VirtIODevice *vdev, + const uint8_t *config) +{ + VirtIOSCSIConfig *scsiconf = (VirtIOSCSIConfig *)config; + VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(vdev); + + if ((uint32_t)virtio_ldl_p(vdev, &scsiconf->sense_size) != vs->sense_size || + (uint32_t)virtio_ldl_p(vdev, &scsiconf->cdb_size) != vs->cdb_size) { + error_report("vhost-user-scsi doesn't allow sense or CDB sizes change"); + exit(1); + } +} + +static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status) +{ + VHostUserSCSI *s = (VHostUserSCSI *)vdev; + bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running; + + if (s->dev.started == start) { + return; + } + + if (start) { + int ret; + + ret = vhost_user_scsi_start(s); + if (ret < 0) { + error_report("unable to start vhost-user-scsi: %s", strerror(-ret)); + exit(1); + } + } else { + vhost_user_scsi_stop(s); + } +} + +static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq) +{ +} + +static void vhost_user_scsi_save(QEMUFile *f, void *opaque) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(opaque); + virtio_save(vdev, f); +} + +static int vhost_user_scsi_load(QEMUFile *f, void *opaque, int version_id) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(opaque); + return virtio_load(vdev, f, version_id); +} + +static void vhost_user_scsi_realize(DeviceState *dev, Error **errp) +{ + VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev); + VHostUserSCSI *s = VHOST_USER_SCSI(dev); + static int vhost_user_scsi_id; + Error *err = NULL; + int ret; + + if (!vs->conf.chardev.chr) { + error_setg(errp, "vhost-user-scsi: missing chardev"); + return; + } + + virtio_scsi_common_realize(dev, &err, vhost_dummy_handle_output, + vhost_dummy_handle_output, + vhost_dummy_handle_output); + if (err != NULL) { + error_propagate(errp, err); + return; + } + + s->dev.nvqs = VHOST_USER_SCSI_VQ_NUM_FIXED + vs->conf.num_queues; + s->dev.vqs = g_new(struct vhost_virtqueue, s->dev.nvqs); + s->dev.vq_index = 0; + s->dev.backend_features = 0; + + ret = vhost_dev_init(&s->dev, (void *)&vs->conf.chardev, + VHOST_BACKEND_TYPE_USER, 0); + if (ret < 0) { + error_setg(errp, "vhost-user-scsi: vhost initialization failed: %s", + strerror(-ret)); + return; + } + + /* Channel and lun both are 0 for bootable vhost-user-scsi disk */ + s->channel = 0; + s->lun = 0; + s->target = vs->conf.boot_tpgt; + + register_savevm(dev, "vhost-user-scsi", vhost_user_scsi_id++, 1, + vhost_user_scsi_save, vhost_user_scsi_load, s); +} + +static void vhost_user_scsi_unrealize(DeviceState *dev, Error **errp) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(dev); + VHostUserSCSI *s = VHOST_USER_SCSI(dev); + + /* This will stop the vhost backend. */ + vhost_user_scsi_set_status(vdev, 0); + + vhost_dev_cleanup(&s->dev); + g_free(s->dev.vqs); + + virtio_scsi_common_unrealize(dev, errp); +} + +/* + * Implementation of an interface to adjust firmware path + * for the bootindex property handling. + */ +static char *vhost_user_scsi_get_fw_dev_path(FWPathProvider *p, BusState *bus, + DeviceState *dev) +{ + VHostUserSCSI *s = VHOST_USER_SCSI(dev); + /* format: /channel@<channel>/vhost-user-scsi@<target>,<lun> */ + return g_strdup_printf("/channel@%x/%s@%x,%x", s->channel, + qdev_fw_name(dev), s->target, s->lun); +} + +static Property vhost_user_scsi_properties[] = { + DEFINE_PROP_CHR("chardev", VHostUserSCSI, + parent_obj.conf.chardev), + DEFINE_PROP_UINT32("boot_tpgt", VHostUserSCSI, + parent_obj.conf.boot_tpgt, 0), + DEFINE_PROP_UINT32("num_queues", VHostUserSCSI, + parent_obj.conf.num_queues, 1), + DEFINE_PROP_UINT32("max_sectors", VHostUserSCSI, + parent_obj.conf.max_sectors, 0xFFFF), + DEFINE_PROP_UINT32("cmd_per_lun", VHostUserSCSI, + parent_obj.conf.cmd_per_lun, 128), + DEFINE_PROP_END_OF_LIST(), +}; + +static void vhost_user_scsi_class_init(ObjectClass *klass, void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); + FWPathProviderClass *fwc = FW_PATH_PROVIDER_CLASS(klass); + + dc->props = vhost_user_scsi_properties; + set_bit(DEVICE_CATEGORY_STORAGE, dc->categories); + vdc->realize = vhost_user_scsi_realize; + vdc->unrealize = vhost_user_scsi_unrealize; + vdc->get_features = vhost_user_scsi_get_features; + vdc->set_config = vhost_user_scsi_set_config; + vdc->set_status = vhost_user_scsi_set_status; + fwc->get_dev_path = vhost_user_scsi_get_fw_dev_path; +} + +static void vhost_user_scsi_instance_init(Object *obj) +{ + VHostUserSCSI *dev = VHOST_USER_SCSI(obj); + VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(obj); + + // Add the bootindex property for this object + device_add_bootindex_property(obj, &dev->bootindex, "bootindex", NULL, + DEVICE(dev), NULL); + + // Set boot index according the the device config + object_property_set_int(obj, vs->conf.bootindex, "bootindex", NULL); +} + +static const TypeInfo vhost_user_scsi_info = { + .name = TYPE_VHOST_USER_SCSI, + .parent = TYPE_VIRTIO_SCSI_COMMON, + .instance_size = sizeof(VHostUserSCSI), + .class_init = vhost_user_scsi_class_init, + .instance_init = vhost_user_scsi_instance_init, + .interfaces = (InterfaceInfo[]) { + { TYPE_FW_PATH_PROVIDER }, + { } + }, +}; + +static void virtio_register_types(void) +{ + type_register_static(&vhost_user_scsi_info); +} + +type_init(virtio_register_types) diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index 06831de..b996d37 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -2098,6 +2098,61 @@ static const TypeInfo vhost_scsi_pci_info = { }; #endif +/* vhost-user-scsi-pci */ +#ifdef CONFIG_VHOST_USER_SCSI +static Property vhost_user_scsi_pci_properties[] = { + DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, + DEV_NVECTORS_UNSPECIFIED), + DEFINE_PROP_END_OF_LIST(), +}; + +static void vhost_user_scsi_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp) +{ + VHostUserSCSIPCI *dev = VHOST_USER_SCSI_PCI(vpci_dev); + DeviceState *vdev = DEVICE(&dev->vdev); + VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(vdev); + + if (vpci_dev->nvectors == DEV_NVECTORS_UNSPECIFIED) { + vpci_dev->nvectors = vs->conf.num_queues + 3; + } + + qdev_set_parent_bus(vdev, BUS(&vpci_dev->bus)); + object_property_set_bool(OBJECT(vdev), true, "realized", errp); +} + +static void vhost_user_scsi_pci_class_init(ObjectClass *klass, void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass); + PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass); + k->realize = vhost_user_scsi_pci_realize; + set_bit(DEVICE_CATEGORY_STORAGE, dc->categories); + dc->props = vhost_user_scsi_pci_properties; + pcidev_k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET; + pcidev_k->device_id = PCI_DEVICE_ID_VIRTIO_SCSI; + pcidev_k->revision = 0x00; + pcidev_k->class_id = PCI_CLASS_STORAGE_SCSI; +} + +static void vhost_user_scsi_pci_instance_init(Object *obj) +{ + VHostUserSCSIPCI *dev = VHOST_USER_SCSI_PCI(obj); + + virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev), + TYPE_VHOST_USER_SCSI); + object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev), + "bootindex", &error_abort); +} + +static const TypeInfo vhost_user_scsi_pci_info = { + .name = TYPE_VHOST_USER_SCSI_PCI, + .parent = TYPE_VIRTIO_PCI, + .instance_size = sizeof(VHostUserSCSIPCI), + .instance_init = vhost_user_scsi_pci_instance_init, + .class_init = vhost_user_scsi_pci_class_init, +}; +#endif /* CONFIG_VHOST_USER_SCSI */ + /* vhost-vsock-pci */ #ifdef CONFIG_VHOST_VSOCK @@ -2577,6 +2632,9 @@ static void virtio_pci_register_types(void) #ifdef CONFIG_VHOST_SCSI type_register_static(&vhost_scsi_pci_info); #endif +#ifdef CONFIG_VHOST_USER_SCSI + type_register_static(&vhost_user_scsi_pci_info); +#endif #ifdef CONFIG_VHOST_VSOCK type_register_static(&vhost_vsock_pci_info); #endif diff --git a/hw/virtio/virtio-pci.h b/hw/virtio/virtio-pci.h index b4edea6..404b2c4 100644 --- a/hw/virtio/virtio-pci.h +++ b/hw/virtio/virtio-pci.h @@ -31,6 +31,9 @@ #ifdef CONFIG_VHOST_SCSI #include "hw/virtio/vhost-scsi.h" #endif +#ifdef CONFIG_VHOST_USER_SCSI +#include "hw/virtio/vhost-user-scsi.h" +#endif #ifdef CONFIG_VHOST_VSOCK #include "hw/virtio/vhost-vsock.h" #endif @@ -42,6 +45,7 @@ typedef struct VirtIOBalloonPCI VirtIOBalloonPCI; typedef struct VirtIOSerialPCI VirtIOSerialPCI; typedef struct VirtIONetPCI VirtIONetPCI; typedef struct VHostSCSIPCI VHostSCSIPCI; +typedef struct VHostUserSCSIPCI VHostUserSCSIPCI; typedef struct VirtIORngPCI VirtIORngPCI; typedef struct VirtIOInputPCI VirtIOInputPCI; typedef struct VirtIOInputHIDPCI VirtIOInputHIDPCI; @@ -212,6 +216,17 @@ struct VHostSCSIPCI { }; #endif +#ifdef CONFIG_VHOST_USER_SCSI +#define TYPE_VHOST_USER_SCSI_PCI "vhost-user-scsi-pci" +#define VHOST_USER_SCSI_PCI(obj) \ + OBJECT_CHECK(VHostUserSCSIPCI, (obj), TYPE_VHOST_USER_SCSI_PCI) + +struct VHostUserSCSIPCI { + VirtIOPCIProxy parent_obj; + VHostUserSCSI vdev; +}; +#endif + /* * virtio-blk-pci: This extends VirtioPCIProxy. */ diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h new file mode 100644 index 0000000..4a16181 --- /dev/null +++ b/include/hw/virtio/vhost-user-scsi.h @@ -0,0 +1,45 @@ +/* + * vhost-user-scsi host device + * + * Copyright (c) 2016 Nutanix Inc. All rights reserved. + * + * Author: + * Felipe Franciosi <felipe@nutanix.com> + * + * This file is largely based on "vhost-scsi.h" by: + * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com> + * + * This work is licensed under the terms of the GNU LGPL, version 2 or later. + * See the COPYING.LIB file in the top-level directory. + * + */ + +#ifndef VHOST_USER_SCSI_H +#define VHOST_USER_SCSI_H + +#include "qemu-common.h" +#include "hw/qdev.h" +#include "hw/virtio/virtio-scsi.h" +#include "hw/virtio/vhost.h" + +enum vhost_user_scsi_vq_list { + VHOST_USER_SCSI_VQ_CONTROL = 0, + VHOST_USER_SCSI_VQ_EVENT = 1, + VHOST_USER_SCSI_VQ_NUM_FIXED = 2, +}; + +#define TYPE_VHOST_USER_SCSI "vhost-user-scsi" +#define VHOST_USER_SCSI(obj) \ + OBJECT_CHECK(VHostUserSCSI, (obj), TYPE_VHOST_USER_SCSI) + +typedef struct VHostUserSCSI { + VirtIOSCSICommon parent_obj; + + struct vhost_dev dev; + int32_t bootindex; + int channel; + int target; + int lun; +} VHostUserSCSI; + +#endif /* VHOST_USER_SCSI_H */ diff --git a/include/hw/virtio/virtio-scsi.h b/include/hw/virtio/virtio-scsi.h index a1e0cfb..4b97386 100644 --- a/include/hw/virtio/virtio-scsi.h +++ b/include/hw/virtio/virtio-scsi.h @@ -21,6 +21,7 @@ #include "hw/virtio/virtio.h" #include "hw/pci/pci.h" #include "hw/scsi/scsi.h" +#include "sysemu/char.h" #include "sysemu/iothread.h" #define TYPE_VIRTIO_SCSI_COMMON "virtio-scsi-common" @@ -53,6 +54,10 @@ struct VirtIOSCSIConf { char *wwpn; uint32_t boot_tpgt; IOThread *iothread; +#ifdef CONFIG_VHOST_USER_SCSI + CharBackend chardev; + int32_t bootindex; +#endif }; struct VirtIOSCSI;
This commit introduces a vhost-user device for SCSI. This is based on the existing vhost-scsi implementation, but done over vhost-user instead. It also uses a chardev to connect to the backend. Unlike vhost-scsi (today), VMs using vhost-user-scsi can be live migrated. To use it, one must configure Qemu with --enable-vhost-user-scsi and start Qemu with a command line equivalent to: qemu-system-x86_64 \ -chardev socket,id=vus0,path=/tmp/vus.sock \ -device vhost-user-scsi-pci,chardev=vus0,bus=pci.0,addr=0x4 A separate commit presents a sample application linked with libiscsi to provide a backend for vhost-user-scsi. Signed-off-by: Felipe Franciosi <felipe@nutanix.com> --- configure | 10 ++ hw/scsi/Makefile.objs | 1 + hw/scsi/vhost-user-scsi.c | 299 ++++++++++++++++++++++++++++++++++++ hw/virtio/virtio-pci.c | 58 +++++++ hw/virtio/virtio-pci.h | 15 ++ include/hw/virtio/vhost-user-scsi.h | 45 ++++++ include/hw/virtio/virtio-scsi.h | 5 + 7 files changed, 433 insertions(+) create mode 100644 hw/scsi/vhost-user-scsi.c create mode 100644 include/hw/virtio/vhost-user-scsi.h