From patchwork Thu Aug 9 00:51:04 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Asias He X-Patchwork-Id: 1298241 Return-Path: X-Original-To: patchwork-kvm@patchwork.kernel.org Delivered-To: patchwork-process-083081@patchwork2.kernel.org Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by patchwork2.kernel.org (Postfix) with ESMTP id 51A00DF223 for ; Thu, 9 Aug 2012 00:50:02 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1759411Ab2HIAt2 (ORCPT ); Wed, 8 Aug 2012 20:49:28 -0400 Received: from mail-pb0-f46.google.com ([209.85.160.46]:52757 "EHLO mail-pb0-f46.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1758675Ab2HIAt0 (ORCPT ); Wed, 8 Aug 2012 20:49:26 -0400 Received: by pbbrr13 with SMTP id rr13so2295880pbb.19 for ; Wed, 08 Aug 2012 17:49:26 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113; h=from:to:cc:subject:date:message-id:x-mailer; bh=58CzY1vhiZ7Q107xChDMhtrORI9g8rq8zW2Np2NkYfw=; b=kANbqlqHz1k/71M5V9oVhyFRvykzgfRZgsO4CSHBnkYVgT16iagmex/wLazzybWTaO MZW60Ga1NwUlj+9zF2OerrOsp/1cZvZPb0ZL6wRhecBzRRvHVTRhj09WXFigPDPW+zly j5d53LJyPmbo/OiLhJpEDKOHVG5CaYKjzgiGcSEku41aFRtS15XaZsiKgPUXvgvOSzrv hxdzZIoozURm/TlSe+9dUM1YiuC4z85QNXi8nagvCueAciOnyQzulWIU7VrUO0Mcy2zJ Rp3dRypgkSBG8Fp9bzksgGAVxRd8DJoAVzuz6fYH8nGutKL0XSYXwgBdh3cI3n3HKcma u02A== Received: by 10.66.89.36 with SMTP id bl4mr37693298pab.58.1344473365917; Wed, 08 Aug 2012 17:49:25 -0700 (PDT) Received: from hj.localdomain.com ([58.194.229.69]) by mx.google.com with ESMTPS id gj10sm14393232pbc.18.2012.08.08.17.49.22 (version=TLSv1/SSLv3 cipher=OTHER); Wed, 08 Aug 2012 17:49:25 -0700 (PDT) From: Asias He To: Pekka Enberg Cc: Sasha Levin , Ingo Molnar , Cyrill Gorcunov , kvm@vger.kernel.org, "Nicholas A. Bellinger" , Stefan Hajnoczi , Paolo Bonzini Subject: [PATCH] kvm tools: Add initial virtio-scsi support Date: Thu, 9 Aug 2012 08:51:04 +0800 Message-Id: <1344473464-17267-1-git-send-email-asias.hejun@gmail.com> X-Mailer: git-send-email 1.7.11.2 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org This patch brings virito-scsi support to kvm tool. With the introduce of tcm_vhost (vhost-scsi) tcm_vhost: Initial merge for vhost level target fabric driver we can implement virito-scsi by simply having vhost-scsi to handle the SCSI command. Howto use: 1) Setup the tcm_vhost target through /sys/kernel/config [Stefan Hajnoczi, Thanks for the script to setup tcm_vhost] ** Setup wwpn and tpgt $ wwpn="naa.0" $ tpgt=/sys/kernel/config/target/vhost/$wwpn/tpgt_0 $ nexus=$tpgt/nexus $ mkdir -p $tpgt $ echo -n $wwpn > $nexus ** Setup lun using /dev/ram $ n=0 $ lun=$tpgt/lun/lun_${n} $ data=/sys/kernel/config/target/core/iblock_0/data_${n} $ ram=/dev/ram${n} $ mkdir -p $lun $ mkdir -p $data $ echo -n udev_path=${ram} > $data/control $ echo -n 1 > $data/enable $ ln -s $data $lun 2) Run kvm tool with the new disk option '-d scsi:$wwpn:$tpgt', e.g $ lkvm run -k /boot/bzImage -d ~/img/sid.img -d scsi:naa.0:0 Signed-off-by: Asias He Cc: Nicholas A. Bellinger Cc: Stefan Hajnoczi Cc: Paolo Bonzini --- tools/kvm/Makefile | 1 + tools/kvm/builtin-run.c | 25 +++ tools/kvm/disk/core.c | 14 ++ tools/kvm/include/kvm/disk-image.h | 4 + tools/kvm/include/kvm/virtio-pci-dev.h | 1 + tools/kvm/include/kvm/virtio-scsi.h | 11 ++ tools/kvm/include/linux/compiler.h | 1 + tools/kvm/virtio/blk.c | 2 + tools/kvm/virtio/scsi.c | 332 +++++++++++++++++++++++++++++++++ 9 files changed, 391 insertions(+) create mode 100644 tools/kvm/include/kvm/virtio-scsi.h create mode 100644 tools/kvm/virtio/scsi.c diff --git a/tools/kvm/Makefile b/tools/kvm/Makefile index f9e1ec1..749956a 100644 --- a/tools/kvm/Makefile +++ b/tools/kvm/Makefile @@ -55,6 +55,7 @@ OBJS += mmio.o OBJS += pci.o OBJS += term.o OBJS += virtio/blk.o +OBJS += virtio/scsi.o OBJS += virtio/console.o OBJS += virtio/core.o OBJS += virtio/net.o diff --git a/tools/kvm/builtin-run.c b/tools/kvm/builtin-run.c index a36bd00..c3ec469 100644 --- a/tools/kvm/builtin-run.c +++ b/tools/kvm/builtin-run.c @@ -8,6 +8,7 @@ #include "kvm/framebuffer.h" #include "kvm/disk-image.h" #include "kvm/threadpool.h" +#include "kvm/virtio-scsi.h" #include "kvm/virtio-blk.h" #include "kvm/virtio-net.h" #include "kvm/virtio-rng.h" @@ -171,6 +172,19 @@ static int img_name_parser(const struct option *opt, const char *arg, int unset) disk_image[image_count].filename = arg; cur = arg; + + if (strncmp(arg, "scsi:", 5) == 0) { + sep = strstr(arg, ":"); + if (sep) + disk_image[image_count].wwpn = sep + 1; + sep = strstr(sep + 1, ":"); + if (sep) { + *sep = 0; + disk_image[image_count].tpgt = sep + 1; + } + cur = sep + 1; + } + do { sep = strstr(cur, ","); if (sep) { @@ -1184,6 +1198,13 @@ static int kvm_cmd_run_init(int argc, const char **argv) goto fail; } + r = virtio_scsi_init(kvm); + if (r < 0) { + pr_err("virtio_scsi_init() failed with error %d\n", r); + goto fail; + } + + if (active_console == CONSOLE_VIRTIO) virtio_console__init(kvm); @@ -1332,6 +1353,10 @@ static void kvm_cmd_run_exit(int guest_ret) fb__stop(); + r = virtio_scsi_exit(kvm); + if (r < 0) + pr_warning("virtio_scsi_exit() failed with error %d\n", r); + r = virtio_blk__exit(kvm); if (r < 0) pr_warning("virtio_blk__exit() failed with error %d\n", r); diff --git a/tools/kvm/disk/core.c b/tools/kvm/disk/core.c index 621c940..f7e2c7f 100644 --- a/tools/kvm/disk/core.c +++ b/tools/kvm/disk/core.c @@ -122,6 +122,8 @@ struct disk_image **disk_image__open_all(struct disk_image_params *params, int c { struct disk_image **disks; const char *filename; + const char *wwpn; + const char *tpgt; bool readonly; bool direct; void *err; @@ -140,6 +142,18 @@ struct disk_image **disk_image__open_all(struct disk_image_params *params, int c filename = params[i].filename; readonly = params[i].readonly; direct = params[i].direct; + wwpn = params[i].wwpn; + tpgt = params[i].tpgt; + + if (wwpn) { + disks[i] = malloc(sizeof(struct disk_image)); + if (!disks[i]) + return ERR_PTR(-ENOMEM); + disks[i]->wwpn = wwpn; + disks[i]->tpgt = tpgt; + continue; + } + if (!filename) continue; diff --git a/tools/kvm/include/kvm/disk-image.h b/tools/kvm/include/kvm/disk-image.h index 7ae17f8..54e4047 100644 --- a/tools/kvm/include/kvm/disk-image.h +++ b/tools/kvm/include/kvm/disk-image.h @@ -41,6 +41,8 @@ struct disk_image_operations { struct disk_image_params { const char *filename; + const char *wwpn; + const char *tpgt; bool readonly; bool direct; }; @@ -57,6 +59,8 @@ struct disk_image { #ifdef CONFIG_HAS_AIO io_context_t ctx; #endif + const char *wwpn; + const char *tpgt; }; struct disk_image *disk_image__open(const char *filename, bool readonly, bool direct); diff --git a/tools/kvm/include/kvm/virtio-pci-dev.h b/tools/kvm/include/kvm/virtio-pci-dev.h index 7ceb125..48ae018 100644 --- a/tools/kvm/include/kvm/virtio-pci-dev.h +++ b/tools/kvm/include/kvm/virtio-pci-dev.h @@ -13,6 +13,7 @@ #define PCI_DEVICE_ID_VIRTIO_CONSOLE 0x1003 #define PCI_DEVICE_ID_VIRTIO_RNG 0x1004 #define PCI_DEVICE_ID_VIRTIO_BLN 0x1005 +#define PCI_DEVICE_ID_VIRTIO_SCSI 0x1008 #define PCI_DEVICE_ID_VIRTIO_9P 0x1009 #define PCI_DEVICE_ID_VESA 0x2000 #define PCI_DEVICE_ID_PCI_SHMEM 0x0001 diff --git a/tools/kvm/include/kvm/virtio-scsi.h b/tools/kvm/include/kvm/virtio-scsi.h new file mode 100644 index 0000000..d64aa7e --- /dev/null +++ b/tools/kvm/include/kvm/virtio-scsi.h @@ -0,0 +1,11 @@ +#ifndef KVM__SCSI_VIRTIO_H +#define KVM__SCSI_VIRTIO_H + +#include "kvm/disk-image.h" + +struct kvm; + +int virtio_scsi_init(struct kvm *kvm); +int virtio_scsi_exit(struct kvm *kvm); + +#endif /* KVM__SCSI_VIRTIO_H */ diff --git a/tools/kvm/include/linux/compiler.h b/tools/kvm/include/linux/compiler.h index b9c5346..898420b 100644 --- a/tools/kvm/include/linux/compiler.h +++ b/tools/kvm/include/linux/compiler.h @@ -11,6 +11,7 @@ #endif #define __used __attribute__((__unused__)) +#define __packed __attribute__((packed)) #define __iomem #define __force #define __must_check diff --git a/tools/kvm/virtio/blk.c b/tools/kvm/virtio/blk.c index 1fb969f..740442a 100644 --- a/tools/kvm/virtio/blk.c +++ b/tools/kvm/virtio/blk.c @@ -290,6 +290,8 @@ int virtio_blk__init(struct kvm *kvm) int i, r = 0; for (i = 0; i < kvm->nr_disks; i++) { + if (kvm->disks[i]->wwpn) + continue; r = virtio_blk__init_one(kvm, kvm->disks[i]); if (r < 0) goto cleanup; diff --git a/tools/kvm/virtio/scsi.c b/tools/kvm/virtio/scsi.c new file mode 100644 index 0000000..5bcb00c --- /dev/null +++ b/tools/kvm/virtio/scsi.c @@ -0,0 +1,332 @@ +#include "kvm/virtio-scsi.h" +#include "kvm/virtio-pci-dev.h" +#include "kvm/disk-image.h" +#include "kvm/kvm.h" +#include "kvm/pci.h" +#include "kvm/ioeventfd.h" +#include "kvm/guest_compat.h" +#include "kvm/virtio-pci.h" +#include "kvm/virtio.h" + +#include +#include +#include + +/*----------------------------------------------------*/ +/* TODO: Remove this when tcm_vhost goes upstream */ +#define TRANSPORT_IQN_LEN 224 +#define VHOST_SCSI_ABI_VERSION 0 +struct vhost_scsi_target { + int abi_version; + unsigned char vhost_wwpn[TRANSPORT_IQN_LEN]; + unsigned short vhost_tpgt; +}; +/* VHOST_SCSI specific defines */ +#define VHOST_SCSI_SET_ENDPOINT _IOW(VHOST_VIRTIO, 0x40, struct vhost_scsi_target) +#define VHOST_SCSI_CLEAR_ENDPOINT _IOW(VHOST_VIRTIO, 0x41, struct vhost_scsi_target) +#define VHOST_SCSI_GET_ABI_VERSION _IOW(VHOST_VIRTIO, 0x42, struct vhost_scsi_target) +/*----------------------------------------------------*/ + + +#define VIRTIO_SCSI_QUEUE_SIZE 128 +#define NUM_VIRT_QUEUES 3 + +static LIST_HEAD(sdevs); +static int compat_id = -1; + +struct scsi_dev { + struct virt_queue vqs[NUM_VIRT_QUEUES]; + struct virtio_scsi_config scsi_config; + struct vhost_scsi_target target; + u32 features; + int vhost_fd; + struct virtio_device vdev; + struct list_head list; + struct kvm *kvm; +}; + +static void set_config(struct kvm *kvm, void *dev, u8 data, u32 offset) +{ + struct scsi_dev *sdev = dev; + + ((u8 *)(&sdev->scsi_config))[offset] = data; +} + +static u8 get_config(struct kvm *kvm, void *dev, u32 offset) +{ + struct scsi_dev *sdev = dev; + + return ((u8 *)(&sdev->scsi_config))[offset]; +} + +static u32 get_host_features(struct kvm *kvm, void *dev) +{ + return 1UL << VIRTIO_RING_F_EVENT_IDX | + 1UL << VIRTIO_RING_F_INDIRECT_DESC; +} + +static void set_guest_features(struct kvm *kvm, void *dev, u32 features) +{ + struct scsi_dev *sdev = dev; + + sdev->features = features; +} + +static int init_vq(struct kvm *kvm, void *dev, u32 vq, u32 pfn) +{ + struct vhost_vring_state state = { .index = vq }; + struct vhost_vring_addr addr; + struct scsi_dev *sdev = dev; + struct virt_queue *queue; + void *p; + int r; + + compat__remove_message(compat_id); + + queue = &sdev->vqs[vq]; + queue->pfn = pfn; + p = guest_pfn_to_host(kvm, queue->pfn); + + vring_init(&queue->vring, VIRTIO_SCSI_QUEUE_SIZE, p, VIRTIO_PCI_VRING_ALIGN); + + if (sdev->vhost_fd == 0) + return 0; + + state.num = queue->vring.num; + r = ioctl(sdev->vhost_fd, VHOST_SET_VRING_NUM, &state); + if (r < 0) + die_perror("VHOST_SET_VRING_NUM failed"); + state.num = 0; + r = ioctl(sdev->vhost_fd, VHOST_SET_VRING_BASE, &state); + if (r < 0) + die_perror("VHOST_SET_VRING_BASE failed"); + + addr = (struct vhost_vring_addr) { + .index = vq, + .desc_user_addr = (u64)(unsigned long)queue->vring.desc, + .avail_user_addr = (u64)(unsigned long)queue->vring.avail, + .used_user_addr = (u64)(unsigned long)queue->vring.used, + }; + + r = ioctl(sdev->vhost_fd, VHOST_SET_VRING_ADDR, &addr); + if (r < 0) + die_perror("VHOST_SET_VRING_ADDR failed"); + + return 0; +} + +static void notify_vq_gsi(struct kvm *kvm, void *dev, u32 vq, u32 gsi) +{ + struct vhost_vring_file file; + struct scsi_dev *sdev = dev; + struct kvm_irqfd irq; + int r; + + if (sdev->vhost_fd == 0) + return; + + irq = (struct kvm_irqfd) { + .gsi = gsi, + .fd = eventfd(0, 0), + }; + file = (struct vhost_vring_file) { + .index = vq, + .fd = irq.fd, + }; + + r = ioctl(kvm->vm_fd, KVM_IRQFD, &irq); + if (r < 0) + die_perror("KVM_IRQFD failed"); + + r = ioctl(sdev->vhost_fd, VHOST_SET_VRING_CALL, &file); + if (r < 0) + die_perror("VHOST_SET_VRING_CALL failed"); + + if (vq > 0) + return; + + r = ioctl(sdev->vhost_fd, VHOST_SCSI_SET_ENDPOINT, &sdev->target); + if (r != 0) + die("VHOST_SCSI_SET_ENDPOINT failed %d", errno); +} + +static void notify_vq_eventfd(struct kvm *kvm, void *dev, u32 vq, u32 efd) +{ + struct scsi_dev *sdev = dev; + struct vhost_vring_file file = { + .index = vq, + .fd = efd, + }; + int r; + + if (sdev->vhost_fd == 0) + return; + + r = ioctl(sdev->vhost_fd, VHOST_SET_VRING_KICK, &file); + if (r < 0) + die_perror("VHOST_SET_VRING_KICK failed"); +} + +static int notify_vq(struct kvm *kvm, void *dev, u32 vq) +{ + return 0; +} + +static int get_pfn_vq(struct kvm *kvm, void *dev, u32 vq) +{ + struct scsi_dev *sdev = dev; + + return sdev->vqs[vq].pfn; +} + +static int get_size_vq(struct kvm *kvm, void *dev, u32 vq) +{ + return VIRTIO_SCSI_QUEUE_SIZE; +} + +static int set_size_vq(struct kvm *kvm, void *dev, u32 vq, int size) +{ + return size; +} + +static struct virtio_ops scsi_dev_virtio_ops = (struct virtio_ops) { + .set_config = set_config, + .get_config = get_config, + .get_host_features = get_host_features, + .set_guest_features = set_guest_features, + .init_vq = init_vq, + .get_pfn_vq = get_pfn_vq, + .get_size_vq = get_size_vq, + .set_size_vq = set_size_vq, + .notify_vq = notify_vq, + .notify_vq_gsi = notify_vq_gsi, + .notify_vq_eventfd = notify_vq_eventfd, +}; + +static void virtio_scsi_vhost_init(struct kvm *kvm, struct scsi_dev *sdev) +{ + struct vhost_memory *mem; + u64 features; + int r; + + sdev->vhost_fd = open("/dev/vhost-scsi", O_RDWR); + if (sdev->vhost_fd < 0) + die_perror("Failed openning vhost-scsi device"); + + mem = calloc(1, sizeof(*mem) + sizeof(struct vhost_memory_region)); + if (mem == NULL) + die("Failed allocating memory for vhost memory map"); + + mem->nregions = 1; + mem->regions[0] = (struct vhost_memory_region) { + .guest_phys_addr = 0, + .memory_size = kvm->ram_size, + .userspace_addr = (unsigned long)kvm->ram_start, + }; + + r = ioctl(sdev->vhost_fd, VHOST_SET_OWNER); + if (r != 0) + die_perror("VHOST_SET_OWNER failed"); + + r = ioctl(sdev->vhost_fd, VHOST_GET_FEATURES, &features); + if (r != 0) + die_perror("VHOST_GET_FEATURES failed"); + + r = ioctl(sdev->vhost_fd, VHOST_SET_FEATURES, &features); + if (r != 0) + die_perror("VHOST_SET_FEATURES failed"); + r = ioctl(sdev->vhost_fd, VHOST_SET_MEM_TABLE, mem); + if (r != 0) + die_perror("VHOST_SET_MEM_TABLE failed"); + + sdev->vdev.use_vhost = true; + + free(mem); +} + + +static int virtio_scsi_init_one(struct kvm *kvm, struct disk_image *disk) +{ + struct scsi_dev *sdev; + + if (!disk) + return -EINVAL; + + sdev = calloc(1, sizeof(struct scsi_dev)); + if (sdev == NULL) + return -ENOMEM; + + *sdev = (struct scsi_dev) { + .scsi_config = (struct virtio_scsi_config) { + .num_queues = NUM_VIRT_QUEUES - 2, + .seg_max = VIRTIO_SCSI_CDB_SIZE - 2, + .max_sectors = 65535, + .cmd_per_lun = 128, + .sense_size = VIRTIO_SCSI_SENSE_SIZE, + .cdb_size = VIRTIO_SCSI_CDB_SIZE, + .max_channel = 0, + .max_target = 0, + .max_lun = 16383, + .event_info_size = sizeof(struct virtio_scsi_event), + }, + .kvm = kvm, + }; + + strncpy((char *)&sdev->target.vhost_wwpn, disk->wwpn, sizeof(sdev->target.vhost_wwpn)); + sdev->target.vhost_tpgt = strtol(disk->tpgt, NULL, 0); + + virtio_init(kvm, sdev, &sdev->vdev, &scsi_dev_virtio_ops, + VIRTIO_PCI, PCI_DEVICE_ID_VIRTIO_SCSI, VIRTIO_ID_SCSI, PCI_CLASS_BLK); + + list_add_tail(&sdev->list, &sdevs); + + virtio_scsi_vhost_init(kvm, sdev); + + if (compat_id == -1) + compat_id = virtio_compat_add_message("virtio-scsi", "CONFIG_VIRTIO_SCSI"); + + return 0; +} + +static int virtio_scsi_exit_one(struct kvm *kvm, struct scsi_dev *sdev) +{ + int r; + + r = ioctl(sdev->vhost_fd, VHOST_SCSI_CLEAR_ENDPOINT, &sdev->target); + if (r != 0) + die("VHOST_SCSI_CLEAR_ENDPOINT failed %d", errno); + + list_del(&sdev->list); + free(sdev); + + return 0; +} + +int virtio_scsi_init(struct kvm *kvm) +{ + int i, r = 0; + + for (i = 0; i < kvm->nr_disks; i++) { + if (!kvm->disks[i]->wwpn) + continue; + r = virtio_scsi_init_one(kvm, kvm->disks[i]); + if (r < 0) + goto cleanup; + } + + return 0; +cleanup: + return virtio_scsi_exit(kvm); +} + +int virtio_scsi_exit(struct kvm *kvm) +{ + while (!list_empty(&sdevs)) { + struct scsi_dev *sdev; + + sdev = list_first_entry(&sdevs, struct scsi_dev, list); + virtio_scsi_exit_one(kvm, sdev); + } + + return 0; +}