@@ -2871,6 +2871,8 @@ F: hw/proxy/qemu-proxy.c
F: include/hw/proxy/qemu-proxy.h
F: include/hw/proxy/memory-sync.h
F: hw/proxy/memory-sync.c
+F: include/remote/iohub.h
+F: remote/iohub.c
Build and test automation
-------------------------
@@ -129,6 +129,7 @@ obj-y += disas.o
obj-$(call notempty,$(TARGET_XML_FILES)) += gdbstub-xml.o
ifeq ($(TARGET_NAME)-$(CONFIG_MPQEMU)-$(CONFIG_USER_ONLY), x86_64-y-)
obj-$(CONFIG_MPQEMU) += hw/proxy/memory-sync.o
+obj-$(CONFIG_MPQEMU) += hw/proxy/qemu-proxy.o
endif
LIBS := $(libs_cpu) $(LIBS)
@@ -45,8 +45,6 @@ endif
common-obj-y += $(devices-dirs-y)
obj-y += $(devices-dirs-y)
-common-obj-$(CONFIG_MPQEMU) += proxy/
-
remote-pci-obj-$(CONFIG_MPQEMU) += core/
remote-pci-obj-$(CONFIG_MPQEMU) += block/
remote-pci-obj-$(CONFIG_MPQEMU) += pci/
deleted file mode 100644
@@ -1 +0,0 @@
-common-obj-$(CONFIG_MPQEMU) += qemu-proxy.o
@@ -15,6 +15,9 @@
#include "hw/pci/pci.h"
#include "hw/proxy/memory-sync.h"
#include "qom/object.h"
+#include "qemu/event_notifier.h"
+#include "sysemu/kvm.h"
+#include "util/event_notifier-posix.c"
static int config_op_send(PCIProxyDev *dev, uint32_t addr, uint32_t *val, int l,
unsigned int op)
@@ -75,6 +78,53 @@ static void pci_proxy_write_config(PCIDevice *d, uint32_t addr, uint32_t val,
config_op_send(PCI_PROXY_DEV(d), addr, &val, l, PCI_CONFIG_WRITE);
}
+static void proxy_intx_update(PCIDevice *pci_dev)
+{
+ PCIProxyDev *dev = PCI_PROXY_DEV(pci_dev);
+ PCIINTxRoute route;
+ int pin = pci_get_byte(pci_dev->config + PCI_INTERRUPT_PIN) - 1;
+
+ if (dev->irqfd.fd) {
+ dev->irqfd.flags = KVM_IRQFD_FLAG_DEASSIGN;
+ (void) kvm_vm_ioctl(kvm_state, KVM_IRQFD, &dev->irqfd);
+ memset(&dev->irqfd, 0, sizeof(struct kvm_irqfd));
+ }
+
+ route = pci_device_route_intx_to_irq(pci_dev, pin);
+
+ dev->irqfd.fd = event_notifier_get_fd(&dev->intr);
+ dev->irqfd.resamplefd = event_notifier_get_fd(&dev->resample);
+ dev->irqfd.gsi = route.irq;
+ dev->irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE;
+ (void) kvm_vm_ioctl(kvm_state, KVM_IRQFD, &dev->irqfd);
+}
+
+static void setup_irqfd(PCIProxyDev *dev)
+{
+ PCIDevice *pci_dev = PCI_DEVICE(dev);
+ MPQemuMsg msg;
+
+ event_notifier_init(&dev->intr, 0);
+ event_notifier_init(&dev->resample, 0);
+
+ memset(&msg, 0, sizeof(MPQemuMsg));
+ msg.cmd = SET_IRQFD;
+ msg.num_fds = 2;
+ msg.fds[0] = event_notifier_get_fd(&dev->intr);
+ msg.fds[1] = event_notifier_get_fd(&dev->resample);
+ msg.data1.set_irqfd.intx =
+ pci_get_byte(pci_dev->config + PCI_INTERRUPT_PIN) - 1;
+ msg.size = sizeof(msg.data1);
+
+ mpqemu_msg_send(&msg, dev->mpqemu_link->dev);
+
+ memset(&dev->irqfd, 0, sizeof(struct kvm_irqfd));
+
+ proxy_intx_update(pci_dev);
+
+ pci_device_set_intx_routing_notifier(pci_dev, proxy_intx_update);
+}
+
static void proxy_set_socket(Object *obj, const char *str, Error **errp)
{
PCIProxyDev *pdev = PCI_PROXY_DEV(obj);
@@ -144,6 +194,8 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error **errp)
dev->sync = REMOTE_MEM_SYNC(object_new(TYPE_MEMORY_LISTENER));
configure_memory_sync(dev->sync, dev->mpqemu_link);
+
+ setup_irqfd(dev);
}
static void pci_proxy_dev_class_init(ObjectClass *klass, void *data)
@@ -192,6 +192,9 @@
#define PCI_DEVICE_ID_SUN_SIMBA 0x5000
#define PCI_DEVICE_ID_SUN_SABRE 0xa000
+#define PCI_VENDOR_ID_ORACLE 0x108e
+#define PCI_DEVICE_ID_REMOTE_IOHUB 0xb000
+
#define PCI_VENDOR_ID_CMD 0x1095
#define PCI_DEVICE_ID_CMD_646 0x0646
@@ -12,9 +12,12 @@
#include "qemu/osdep.h"
#include "qemu-common.h"
+#include <linux/kvm.h>
+
#include "io/mpqemu-link.h"
#include "hw/pci/pci.h"
#include "hw/proxy/memory-sync.h"
+#include "qemu/event_notifier.h"
#define TYPE_PCI_PROXY_DEV "pci-proxy-dev"
@@ -47,6 +50,11 @@ struct PCIProxyDev {
RemoteMemSync *sync;
+ struct kvm_irqfd irqfd;
+
+ EventNotifier intr;
+ EventNotifier resample;
+
int socket;
ProxyMemoryRegion region[PCI_NUM_REGIONS];
@@ -33,6 +33,8 @@
* SYNC_SYSMEM Shares QEMU's RAM with remote device's RAM
* BAR_WRITE Writes to PCI BAR region
* BAR_READ Reads from PCI BAR region
+ * SET_IRQFD Sets the IRQFD to be used to raise interrupts directly
+ * from remote device
*
* proc_cmd_t enum type to specify the command to be executed on the remote
* device.
@@ -45,6 +47,7 @@ typedef enum {
PCI_CONFIG_READ,
BAR_WRITE,
BAR_READ,
+ SET_IRQFD,
MAX,
} mpqemu_cmd_t;
@@ -81,6 +84,10 @@ typedef struct {
*
*/
+typedef struct {
+ int intx;
+} set_irqfd_msg_t;
+
typedef struct {
mpqemu_cmd_t cmd;
int bytestream;
@@ -90,6 +97,7 @@ typedef struct {
uint64_t u64;
sync_sysmem_msg_t sync_sysmem;
bar_access_msg_t bar_access;
+ set_irqfd_msg_t set_irqfd;
} data1;
int fds[REMOTE_MAX_FDS];
new file mode 100644
@@ -0,0 +1,50 @@
+/*
+ * IO Hub for remote device
+ *
+ * Copyright © 2018, 2020 Oracle and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef REMOTE_IOHUB_H
+#define REMOTE_IOHUB_H
+
+#include <sys/types.h>
+
+#include "qemu/osdep.h"
+#include "hw/pci/pci.h"
+#include "qemu/event_notifier.h"
+#include "qemu/thread-posix.h"
+#include "io/mpqemu-link.h"
+
+#define REMOTE_IOHUB_NB_PIRQS 8
+
+#define REMOTE_IOHUB_DEV 31
+#define REMOTE_IOHUB_FUNC 0
+
+#define TYPE_REMOTE_IOHUB_DEVICE "remote-iohub"
+#define REMOTE_IOHUB_DEVICE(obj) \
+ OBJECT_CHECK(RemoteIOHubState, (obj), TYPE_REMOTE_IOHUB_DEVICE)
+
+typedef struct ResampleToken {
+ void *iohub;
+ int pirq;
+} ResampleToken;
+
+typedef struct RemoteIOHubState {
+ PCIDevice d;
+ uint8_t irq_num[PCI_SLOT_MAX][PCI_NUM_PINS];
+ EventNotifier irqfds[REMOTE_IOHUB_NB_PIRQS];
+ EventNotifier resamplefds[REMOTE_IOHUB_NB_PIRQS];
+ unsigned int irq_level[REMOTE_IOHUB_NB_PIRQS];
+ ResampleToken token[REMOTE_IOHUB_NB_PIRQS];
+ QemuMutex irq_level_lock[REMOTE_IOHUB_NB_PIRQS];
+} RemoteIOHubState;
+
+int remote_iohub_map_irq(PCIDevice *pci_dev, int intx);
+void remote_iohub_set_irq(void *opaque, int pirq, int level);
+void process_set_irqfd_msg(PCIDevice *pci_dev, MPQemuMsg *msg);
+
+#endif
@@ -16,11 +16,13 @@
#include "hw/boards.h"
#include "remote/pcihost.h"
#include "qemu/notify.h"
+#include "remote/iohub.h"
typedef struct RemMachineState {
MachineState parent_obj;
RemPCIHost *host;
+ RemoteIOHubState *iohub;
} RemMachineState;
#define TYPE_REMOTE_MACHINE "remote-machine"
@@ -389,6 +389,7 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)
break;
case BAR_WRITE:
case BAR_READ:
+ case SET_IRQFD:
if (msg->size != sizeof(msg->data1)) {
return false;
}
@@ -2,3 +2,4 @@ remote-pci-obj-$(CONFIG_MPQEMU) += remote-main.o
remote-pci-obj-$(CONFIG_MPQEMU) += pcihost.o
remote-pci-obj-$(CONFIG_MPQEMU) += machine.o
remote-pci-obj-$(CONFIG_MPQEMU) += ../util/machine-notify.o
+remote-pci-obj-$(CONFIG_MPQEMU) += iohub.o
new file mode 100644
@@ -0,0 +1,148 @@
+/*
+ * Remote IO Hub
+ *
+ * Copyright © 2018, 2020 Oracle and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include <sys/types.h>
+
+#include "qemu/osdep.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_ids.h"
+#include "hw/pci/pci_bus.h"
+#include "remote/iohub.h"
+#include "qemu/thread.h"
+#include "hw/boards.h"
+#include "remote/machine.h"
+#include "qemu/main-loop.h"
+
+static void remote_iohub_initfn(Object *obj)
+{
+ RemoteIOHubState *iohub = REMOTE_IOHUB_DEVICE(obj);
+ int slot, intx, pirq;
+
+ memset(&iohub->irqfds, 0, sizeof(iohub->irqfds));
+ memset(&iohub->resamplefds, 0, sizeof(iohub->resamplefds));
+
+ for (slot = 0; slot < PCI_SLOT_MAX; slot++) {
+ for (intx = 0; intx < PCI_NUM_PINS; intx++) {
+ iohub->irq_num[slot][intx] = (slot + intx) % 4 + 4;
+ }
+ }
+
+ for (pirq = 0; pirq < REMOTE_IOHUB_NB_PIRQS; pirq++) {
+ qemu_mutex_init(&iohub->irq_level_lock[pirq]);
+ iohub->irq_level[pirq] = 0;
+ }
+}
+
+static void remote_iohub_class_init(ObjectClass *klass, void *data)
+{
+ PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+ k->vendor_id = PCI_VENDOR_ID_ORACLE;
+ k->device_id = PCI_DEVICE_ID_REMOTE_IOHUB;
+}
+
+static const TypeInfo remote_iohub_info = {
+ .name = TYPE_REMOTE_IOHUB_DEVICE,
+ .parent = TYPE_PCI_DEVICE,
+ .instance_size = sizeof(RemoteIOHubState),
+ .instance_init = remote_iohub_initfn,
+ .class_init = remote_iohub_class_init,
+ .interfaces = (InterfaceInfo[]) {
+ { INTERFACE_CONVENTIONAL_PCI_DEVICE },
+ { }
+ }
+};
+
+static void remote_iohub_register(void)
+{
+ type_register_static(&remote_iohub_info);
+}
+
+type_init(remote_iohub_register);
+
+int remote_iohub_map_irq(PCIDevice *pci_dev, int intx)
+{
+ BusState *bus = qdev_get_parent_bus(&pci_dev->qdev);
+ PCIBus *pci_bus = PCI_BUS(bus);
+ PCIDevice *pci_iohub =
+ pci_bus->devices[PCI_DEVFN(REMOTE_IOHUB_DEV, REMOTE_IOHUB_FUNC)];
+ RemoteIOHubState *iohub = REMOTE_IOHUB_DEVICE(pci_iohub);
+
+ return iohub->irq_num[PCI_SLOT(pci_dev->devfn)][intx];
+}
+
+/*
+ * TODO: Using lock to set the interrupt level could become a
+ * performance bottleneck. Check if atomic arithmetic
+ * is possible.
+ */
+void remote_iohub_set_irq(void *opaque, int pirq, int level)
+{
+ RemoteIOHubState *iohub = opaque;
+
+ assert(pirq >= 0);
+ assert(pirq < REMOTE_IOHUB_NB_PIRQS);
+
+ qemu_mutex_lock(&iohub->irq_level_lock[pirq]);
+
+ if (level) {
+ if (++iohub->irq_level[pirq] == 1) {
+ event_notifier_set(&iohub->irqfds[pirq]);
+ }
+ } else if (iohub->irq_level[pirq] > 0) {
+ iohub->irq_level[pirq]--;
+ }
+
+ qemu_mutex_unlock(&iohub->irq_level_lock[pirq]);
+}
+
+static void intr_resample_handler(void *opaque)
+{
+ ResampleToken *token = opaque;
+ RemoteIOHubState *iohub = token->iohub;
+ int pirq, s;
+
+ pirq = token->pirq;
+
+ s = event_notifier_test_and_clear(&iohub->resamplefds[pirq]);
+
+ assert(s >= 0);
+
+ qemu_mutex_lock(&iohub->irq_level_lock[pirq]);
+
+ if (iohub->irq_level[pirq]) {
+ event_notifier_set(&iohub->irqfds[pirq]);
+ }
+
+ qemu_mutex_unlock(&iohub->irq_level_lock[pirq]);
+}
+
+void process_set_irqfd_msg(PCIDevice *pci_dev, MPQemuMsg *msg)
+{
+ RemMachineState *machine = REMOTE_MACHINE(current_machine);
+ RemoteIOHubState *iohub = machine->iohub;
+ int pirq = remote_iohub_map_irq(pci_dev, msg->data1.set_irqfd.intx);
+
+ assert(msg->num_fds == 2);
+
+ if (event_notifier_get_fd(&iohub->irqfds[pirq]) != -1) {
+ event_notifier_cleanup(&iohub->irqfds[pirq]);
+ event_notifier_cleanup(&iohub->resamplefds[pirq]);
+ memset(&iohub->token[pirq], 0, sizeof(ResampleToken));
+ }
+
+ event_notifier_init_fd(&iohub->irqfds[pirq], msg->fds[0]);
+ event_notifier_init_fd(&iohub->resamplefds[pirq], msg->fds[1]);
+
+ iohub->token[pirq].iohub = iohub;
+ iohub->token[pirq].pirq = pirq;
+
+ qemu_set_fd_handler(msg->fds[1], intr_resample_handler, NULL,
+ &iohub->token[pirq]);
+}
@@ -25,12 +25,16 @@
#include "qemu-common.h"
#include "sysemu/sysemu.h"
#include "qemu/notify.h"
+#include "hw/pci/pci_host.h"
+#include "remote/iohub.h"
static void remote_machine_init(Object *obj)
{
RemMachineState *s = REMOTE_MACHINE(obj);
RemPCIHost *rem_host;
MemoryRegion *system_memory, *system_io, *pci_memory;
+ PCIHostState *pci_host;
+ PCIDevice *pci_dev;
Error *error_abort = NULL;
@@ -67,6 +71,17 @@ static void remote_machine_init(Object *obj)
qemu_mutex_unlock_iothread();
qdev_init_nofail(DEVICE(rem_host));
+
+ pci_host = PCI_HOST_BRIDGE(rem_host);
+ pci_dev = pci_create_simple_multifunction(pci_host->bus,
+ PCI_DEVFN(REMOTE_IOHUB_DEV,
+ REMOTE_IOHUB_FUNC),
+ true, TYPE_REMOTE_IOHUB_DEVICE);
+
+ s->iohub = REMOTE_IOHUB_DEVICE(pci_dev);
+
+ pci_bus_irqs(pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq,
+ s->iohub, REMOTE_IOHUB_NB_PIRQS);
}
static const TypeInfo remote_machine = {
@@ -36,6 +36,7 @@
#include "remote/remote-common.h"
#include "exec/memattrs.h"
#include "exec/address-spaces.h"
+#include "remote/iohub.h"
static void process_msg(GIOCondition cond, MPQemuLinkState *link,
MPQemuChannel *chan);
@@ -242,6 +243,9 @@ static void process_msg(GIOCondition cond, MPQemuLinkState *link,
goto finalize_loop;
}
break;
+ case SET_IRQFD:
+ process_set_irqfd_msg(LINK_TO_DEV(link), msg);
+ break;
default:
error_setg(&err, "Unknown command in %s", print_pid_exec(pid_exec));
goto finalize_loop;