From patchwork Thu Aug 16 13:54:25 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jan Kiszka X-Patchwork-Id: 1332181 Return-Path: X-Original-To: patchwork-kvm@patchwork.kernel.org Delivered-To: patchwork-process-083081@patchwork1.kernel.org Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by patchwork1.kernel.org (Postfix) with ESMTP id F273140214 for ; Thu, 16 Aug 2012 13:55:45 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S932537Ab2HPNzo (ORCPT ); Thu, 16 Aug 2012 09:55:44 -0400 Received: from david.siemens.de ([192.35.17.14]:34236 "EHLO david.siemens.de" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S932405Ab2HPNyw (ORCPT ); Thu, 16 Aug 2012 09:54:52 -0400 Received: from mail1.siemens.de (localhost [127.0.0.1]) by david.siemens.de (8.13.6/8.13.6) with ESMTP id q7GDsZdF007015; Thu, 16 Aug 2012 15:54:35 +0200 Received: from mchn199C.mchp.siemens.de ([139.25.109.49]) by mail1.siemens.de (8.13.6/8.13.6) with ESMTP id q7GDsYJL012893; Thu, 16 Aug 2012 15:54:35 +0200 From: Jan Kiszka To: Avi Kivity , Marcelo Tosatti Cc: kvm@vger.kernel.org, Alex Williamson Subject: [PATCH 11/19] pci-assign: Rework MSI-X route setup Date: Thu, 16 Aug 2012 15:54:25 +0200 Message-Id: X-Mailer: git-send-email 1.7.3.4 In-Reply-To: References: In-Reply-To: References: Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org Use kvm_irqchip_add_msi_route and introduce kvm_irqchip_update_msi_route to set up the required IRQ routes for MSI-X injections. This removes the last direct interaction with the IRQ routing API of the KVM kernel so that we can remove/unexport related services. Signed-off-by: Jan Kiszka --- hw/device-assignment.c | 71 ++++++++++----------------- kvm-all.c | 51 ++++++++++++++++--- kvm-stub.c | 9 --- kvm.h | 5 +-- qemu-kvm.c | 128 ------------------------------------------------ qemu-kvm.h | 22 -------- 6 files changed, 71 insertions(+), 215 deletions(-) diff --git a/hw/device-assignment.c b/hw/device-assignment.c index af8a5aa..7ffd26c 100644 --- a/hw/device-assignment.c +++ b/hw/device-assignment.c @@ -141,8 +141,6 @@ typedef struct AssignedDevice { uint8_t emulate_config_write[PCI_CONFIG_SPACE_SIZE]; int msi_virq_nr; int *msi_virq; - int irq_entries_nr; - struct kvm_irq_routing_entry *entry; MSIXTableEntry *msix_table; target_phys_addr_t msix_table_addr; uint16_t msix_max; @@ -701,7 +699,7 @@ again: static QLIST_HEAD(, AssignedDevice) devs = QLIST_HEAD_INITIALIZER(devs); -static void free_dev_irq_entries(AssignedDevice *dev) +static void free_msi_virqs(AssignedDevice *dev) { int i; @@ -714,15 +712,6 @@ static void free_dev_irq_entries(AssignedDevice *dev) g_free(dev->msi_virq); dev->msi_virq = NULL; dev->msi_virq_nr = 0; - - for (i = 0; i < dev->irq_entries_nr; i++) { - if (dev->entry[i].type) { - kvm_del_routing_entry(&dev->entry[i]); - } - } - g_free(dev->entry); - dev->entry = NULL; - dev->irq_entries_nr = 0; } static void free_assigned_device(AssignedDevice *dev) @@ -778,7 +767,7 @@ static void free_assigned_device(AssignedDevice *dev) close(dev->real_device.config_fd); } - free_dev_irq_entries(dev); + free_msi_virqs(dev); } static void assign_failed_examine(AssignedDevice *dev) @@ -1001,7 +990,7 @@ static void assigned_dev_update_msi(PCIDevice *pci_dev) if (r && r != -ENXIO) perror("assigned_dev_update_msi: deassign irq"); - free_dev_irq_entries(assigned_dev); + free_msi_virqs(assigned_dev); assigned_dev->assigned_irq_type = ASSIGNED_IRQ_NONE; pci_device_set_intx_routing_notifier(pci_dev, NULL); @@ -1046,6 +1035,7 @@ static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev) uint16_t entries_nr = 0; int i, r = 0; MSIXTableEntry *entry = adev->msix_table; + MSIMessage msg; /* Get the usable entry number for allocating */ for (i = 0; i < adev->msix_max; i++, entry++) { @@ -1069,45 +1059,38 @@ static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev) return r; } - free_dev_irq_entries(adev); + free_msi_virqs(adev); - adev->irq_entries_nr = adev->msix_max; - adev->entry = g_malloc0(adev->msix_max * sizeof(*(adev->entry))); + adev->msi_virq_nr = adev->msix_max; + adev->msi_virq = g_malloc(adev->msix_max * sizeof(*adev->msi_virq)); entry = adev->msix_table; for (i = 0; i < adev->msix_max; i++, entry++) { + adev->msi_virq[i] = -1; + if (msix_masked(entry)) { continue; } - r = kvm_get_irq_route_gsi(); - if (r < 0) + msg.address = entry->addr_lo | ((uint64_t)entry->addr_hi << 32); + msg.data = entry->data; + r = kvm_irqchip_add_msi_route(kvm_state, msg); + if (r < 0) { return r; - - adev->entry[i].gsi = r; - adev->entry[i].type = KVM_IRQ_ROUTING_MSI; - adev->entry[i].flags = 0; - adev->entry[i].u.msi.address_lo = entry->addr_lo; - adev->entry[i].u.msi.address_hi = entry->addr_hi; - adev->entry[i].u.msi.data = entry->data; + } + adev->msi_virq[i] = r; DEBUG("MSI-X vector %d, gsi %d, addr %08x_%08x, data %08x\n", i, r, entry->addr_hi, entry->addr_lo, entry->data); - kvm_add_routing_entry(kvm_state, &adev->entry[i]); - r = kvm_device_msix_set_vector(kvm_state, adev->dev_id, i, - adev->entry[i].gsi); + adev->msi_virq[i]); if (r) { fprintf(stderr, "fail to set MSI-X entry! %s\n", strerror(-r)); break; } } - if (r == 0) { - kvm_irqchip_commit_routes(kvm_state); - } - return r; } @@ -1127,13 +1110,13 @@ static void assigned_dev_update_msix(PCIDevice *pci_dev) * MSIX or intends to start. */ if ((assigned_dev->assigned_irq_type == ASSIGNED_IRQ_MSIX) || (ctrl_word & PCI_MSIX_FLAGS_ENABLE)) { - - free_dev_irq_entries(assigned_dev); r = kvm_device_msix_deassign(kvm_state, assigned_dev->dev_id); /* -ENXIO means no assigned irq */ if (r && r != -ENXIO) perror("assigned_dev_update_msix: deassign irq"); + free_msi_virqs(assigned_dev); + assigned_dev->assigned_irq_type = ASSIGNED_IRQ_NONE; pci_device_set_intx_routing_notifier(pci_dev, NULL); } @@ -1147,7 +1130,7 @@ static void assigned_dev_update_msix(PCIDevice *pci_dev) return; } - if (assigned_dev->irq_entries_nr) { + if (assigned_dev->msi_virq_nr > 0) { if (kvm_assign_irq(kvm_state, &assigned_irq_data) < 0) { perror("assigned_dev_enable_msix: assign irq"); return; @@ -1561,27 +1544,25 @@ static void msix_mmio_write(void *opaque, target_phys_addr_t addr, */ } else if (msix_masked(&orig) && !msix_masked(entry)) { /* Vector unmasked */ - if (i >= adev->irq_entries_nr || !adev->entry[i].type) { + if (i >= adev->msi_virq_nr || adev->msi_virq[i] < 0) { /* Previously unassigned vector, start from scratch */ assigned_dev_update_msix(pdev); return; } else { /* Update an existing, previously masked vector */ - struct kvm_irq_routing_entry orig = adev->entry[i]; + MSIMessage msg; int ret; - adev->entry[i].u.msi.address_lo = entry->addr_lo; - adev->entry[i].u.msi.address_hi = entry->addr_hi; - adev->entry[i].u.msi.data = entry->data; + msg.address = entry->addr_lo | + ((uint64_t)entry->addr_hi << 32); + msg.data = entry->data; - ret = kvm_update_routing_entry(&orig, &adev->entry[i]); + ret = kvm_irqchip_update_msi_route(kvm_state, + adev->msi_virq[i], msg); if (ret) { fprintf(stderr, "Error updating irq routing entry (%d)\n", ret); - return; } - - kvm_irqchip_commit_routes(kvm_state); } } } diff --git a/kvm-all.c b/kvm-all.c index 979c8d7..8ab47f1 100644 --- a/kvm-all.c +++ b/kvm-all.c @@ -922,7 +922,7 @@ static void kvm_init_irq_routing(KVMState *s) kvm_arch_init_irq_routing(s); } -void kvm_irqchip_commit_routes(KVMState *s) +static void kvm_irqchip_commit_routes(KVMState *s) { int ret; @@ -931,7 +931,7 @@ void kvm_irqchip_commit_routes(KVMState *s) assert(ret == 0); } -void kvm_add_routing_entry(KVMState *s, +static void kvm_add_routing_entry(KVMState *s, struct kvm_irq_routing_entry *entry) { struct kvm_irq_routing_entry *new; @@ -960,6 +960,30 @@ void kvm_add_routing_entry(KVMState *s, kvm_irqchip_commit_routes(s); } +static int kvm_update_routing_entry(KVMState *s, + struct kvm_irq_routing_entry *new_entry) +{ + struct kvm_irq_routing_entry *entry; + int n; + + for (n = 0; n < s->irq_routes->nr; n++) { + entry = &s->irq_routes->entries[n]; + if (entry->gsi != new_entry->gsi) { + continue; + } + + entry->type = new_entry->type; + entry->flags = new_entry->flags; + entry->u = new_entry->u; + + kvm_irqchip_commit_routes(s); + + return 0; + } + + return -ESRCH; +} + void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin) { struct kvm_irq_routing_entry e; @@ -1122,6 +1146,24 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg) return virq; } +int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg) +{ + struct kvm_irq_routing_entry kroute; + + if (!kvm_irqchip_in_kernel()) { + return -ENOSYS; + } + + kroute.gsi = virq; + kroute.type = KVM_IRQ_ROUTING_MSI; + kroute.flags = 0; + kroute.u.msi.address_lo = (uint32_t)msg.address; + kroute.u.msi.address_hi = msg.address >> 32; + kroute.u.msi.data = msg.data; + + return kvm_update_routing_entry(s, &kroute); +} + static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign) { struct kvm_irqfd irqfd = { @@ -1143,11 +1185,6 @@ static void kvm_init_irq_routing(KVMState *s) { } -int kvm_irqchip_commit_routes(KVMState *s) -{ - return -ENOSYS; -} - void kvm_irqchip_release_virq(KVMState *s, int virq) { } diff --git a/kvm-stub.c b/kvm-stub.c index 6c8d980..94c9ea1 100644 --- a/kvm-stub.c +++ b/kvm-stub.c @@ -122,15 +122,6 @@ int kvm_set_ioeventfd_mmio(int fd, uint32_t adr, uint32_t val, bool assign, uint return -ENOSYS; } -int kvm_get_irq_route_gsi(void) -{ - return -ENOSYS; -} - -void kvm_irqchip_commit_routes(KVMState *s) -{ -} - int kvm_on_sigbus_vcpu(CPUArchState *env, int code, void *addr) { return 1; diff --git a/kvm.h b/kvm.h index cdfbb82..0c09be8 100644 --- a/kvm.h +++ b/kvm.h @@ -273,6 +273,7 @@ int kvm_set_ioeventfd_mmio(int fd, uint32_t adr, uint32_t val, bool assign, int kvm_set_ioeventfd_pio_word(int fd, uint16_t adr, uint16_t val, bool assign); int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg); +int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg); void kvm_irqchip_release_virq(KVMState *s, int virq); int kvm_irqchip_add_irqfd(KVMState *s, int fd, int virq); @@ -280,10 +281,6 @@ int kvm_irqchip_remove_irqfd(KVMState *s, int fd, int virq); int kvm_irqchip_add_irq_notifier(KVMState *s, EventNotifier *n, int virq); int kvm_irqchip_remove_irq_notifier(KVMState *s, EventNotifier *n, int virq); -int kvm_get_irq_route_gsi(void); - -void kvm_irqchip_commit_routes(KVMState *s); - #ifdef NEED_CPU_H #include "qemu-kvm.h" #endif diff --git a/qemu-kvm.c b/qemu-kvm.c index ec1911f..e45e4a7 100644 --- a/qemu-kvm.c +++ b/qemu-kvm.c @@ -57,134 +57,6 @@ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq) #endif #endif -int kvm_del_routing_entry(struct kvm_irq_routing_entry *entry) -{ -#ifdef KVM_CAP_IRQ_ROUTING - KVMState *s = kvm_state; - struct kvm_irq_routing_entry *e, *p; - int i, gsi, found = 0; - - gsi = entry->gsi; - - for (i = 0; i < s->irq_routes->nr; ++i) { - e = &s->irq_routes->entries[i]; - if (e->type == entry->type && e->gsi == gsi) { - switch (e->type) { - case KVM_IRQ_ROUTING_IRQCHIP:{ - if (e->u.irqchip.irqchip == - entry->u.irqchip.irqchip - && e->u.irqchip.pin == entry->u.irqchip.pin) { - p = &s->irq_routes->entries[--s->irq_routes->nr]; - *e = *p; - found = 1; - } - break; - } - case KVM_IRQ_ROUTING_MSI:{ - if (e->u.msi.address_lo == - entry->u.msi.address_lo - && e->u.msi.address_hi == - entry->u.msi.address_hi - && e->u.msi.data == entry->u.msi.data) { - p = &s->irq_routes->entries[--s->irq_routes->nr]; - *e = *p; - found = 1; - } - break; - } - default: - break; - } - if (found) { - /* If there are no other users of this GSI - * mark it available in the bitmap */ - for (i = 0; i < s->irq_routes->nr; i++) { - e = &s->irq_routes->entries[i]; - if (e->gsi == gsi) - break; - } - if (i == s->irq_routes->nr) { - clear_gsi(s, gsi); - } - - return 0; - } - } - } - return -ESRCH; -#else - return -ENOSYS; -#endif -} - -int kvm_update_routing_entry(struct kvm_irq_routing_entry *entry, - struct kvm_irq_routing_entry *newentry) -{ -#ifdef KVM_CAP_IRQ_ROUTING - KVMState *s = kvm_state; - struct kvm_irq_routing_entry *e; - int i; - - if (entry->gsi != newentry->gsi || entry->type != newentry->type) { - return -EINVAL; - } - - for (i = 0; i < s->irq_routes->nr; ++i) { - e = &s->irq_routes->entries[i]; - if (e->type != entry->type || e->gsi != entry->gsi) { - continue; - } - switch (e->type) { - case KVM_IRQ_ROUTING_IRQCHIP: - if (e->u.irqchip.irqchip == entry->u.irqchip.irqchip && - e->u.irqchip.pin == entry->u.irqchip.pin) { - memcpy(&e->u.irqchip, &newentry->u.irqchip, - sizeof e->u.irqchip); - return 0; - } - break; - case KVM_IRQ_ROUTING_MSI: - if (e->u.msi.address_lo == entry->u.msi.address_lo && - e->u.msi.address_hi == entry->u.msi.address_hi && - e->u.msi.data == entry->u.msi.data) { - memcpy(&e->u.msi, &newentry->u.msi, sizeof e->u.msi); - return 0; - } - break; - default: - break; - } - } - return -ESRCH; -#else - return -ENOSYS; -#endif -} - -int kvm_get_irq_route_gsi(void) -{ -#ifdef KVM_CAP_IRQ_ROUTING - KVMState *s = kvm_state; - int max_words = ALIGN(s->gsi_count, 32) / 32; - int i, bit; - uint32_t *buf = s->used_gsi_bitmap; - - /* Return the lowest unused GSI in the bitmap */ - for (i = 0; i < max_words; i++) { - bit = ffs(~buf[i]); - if (!bit) { - continue; - } - - return bit - 1 + i * 32; - } - - return -ENOSPC; -#else - return -ENOSYS; -#endif -} - #if !defined(TARGET_I386) void kvm_arch_init_irq_routing(KVMState *s) { diff --git a/qemu-kvm.h b/qemu-kvm.h index ad628d5..ae7a33c 100644 --- a/qemu-kvm.h +++ b/qemu-kvm.h @@ -43,28 +43,6 @@ */ int kvm_assign_irq(KVMState *s, struct kvm_assigned_irq *assigned_irq); -struct kvm_irq_routing_entry; - -void kvm_add_routing_entry(KVMState *s, struct kvm_irq_routing_entry *entry); - -/*! - * \brief Removes a routing from the temporary irq routing table - * - * Remove a routing to the temporary irq routing table. Nothing is - * committed to the running VM. - */ -int kvm_del_routing_entry(struct kvm_irq_routing_entry *entry); - -/*! - * \brief Updates a routing in the temporary irq routing table - * - * Update a routing in the temporary irq routing table - * with a new value. entry type and GSI can not be changed. - * Nothing is committed to the running VM. - */ -int kvm_update_routing_entry(struct kvm_irq_routing_entry *entry, - struct kvm_irq_routing_entry *newentry); - #endif /* CONFIG_KVM */ #endif