[v2,4/8] kvm tools: add generic device registration mechanism

Commit Message

Will Deacon Nov. 22, 2012, 3:58 p.m. UTC

PCI devices are currently registered into the pci_devices array via the
pci__register function, which can then be indexed later by architecture
code to construct device tree nodes. For MMIO devices, there is no such
utility.

Rather than invent a similar mechanism for MMIO, this patch creates a
global device registration mechanism, which allows the device type to be
specified when registered or indexing a device. Current users of the pci
registration code are migrated to the new infrastructure and virtio MMIO
devices are registered at init time.

As part of the device registration, allocation of the device number is
moved out of irq__register_device and performed when adding the device
header to the relevant bus tree, allowing us to maintain separate device
numberspaces for each bus.

Signed-off-by: Will Deacon <[e-mail]>
---
 tools/kvm/Makefile                  |  1 +
 tools/kvm/devices.c                 | 86 +++++++++++++++++++++++++++++++++++++
 tools/kvm/hw/pci-shmem.c            | 12 ++++--
 tools/kvm/hw/vesa.c                 | 12 ++++--
 tools/kvm/include/kvm/devices.h     | 27 ++++++++++++
 tools/kvm/include/kvm/irq.h         |  2 +-
 tools/kvm/include/kvm/pci.h         |  2 -
 tools/kvm/include/kvm/virtio-mmio.h |  1 +
 tools/kvm/include/kvm/virtio-pci.h  |  2 +
 tools/kvm/pci.c                     | 39 ++++++-----------
 tools/kvm/powerpc/irq.c             | 10 +----
 tools/kvm/powerpc/spapr_pci.c       |  2 +-
 tools/kvm/virtio/mmio.c             | 11 ++++-
 tools/kvm/virtio/pci.c              | 11 +++--
 tools/kvm/x86/irq.c                 |  4 +-
 15 files changed, 169 insertions(+), 53 deletions(-)
 create mode 100644 tools/kvm/devices.c
 create mode 100644 tools/kvm/include/kvm/devices.h

Patch

diff --git a/tools/kvm/Makefile b/tools/kvm/Makefile
index de11060..3f25a14 100644
--- a/tools/kvm/Makefile
+++ b/tools/kvm/Makefile
@@ -50,6 +50,7 @@  OBJS	+= builtin-run.o
 OBJS	+= builtin-setup.o
 OBJS	+= builtin-stop.o
 OBJS	+= builtin-version.o
+OBJS	+= devices.o
 OBJS	+= disk/core.o
 OBJS	+= framebuffer.o
 OBJS	+= guest_compat.o
diff --git a/tools/kvm/devices.c b/tools/kvm/devices.c
new file mode 100644
index 0000000..9f1941d
--- /dev/null
+++ b/tools/kvm/devices.c
@@ -0,0 +1,86 @@ 
+#include "kvm/devices.h"
+#include "kvm/kvm.h"
+
+#include <linux/err.h>
+#include <linux/rbtree.h>
+
+struct device_bus {
+	struct rb_root	root;
+	int		dev_num;
+};
+
+static struct device_bus device_trees[DEVICE_BUS_MAX] = {
+	[0 ... (DEVICE_BUS_MAX - 1)] = { RB_ROOT, 0 },
+};
+
+int device__register(struct device_header *dev)
+{
+	struct device_bus *bus;
+	struct rb_node **node, *parent = NULL;
+
+	if (dev->bus_type >= DEVICE_BUS_MAX) {
+		pr_warning("Ignoring device registration on unknown bus %d\n",
+			   dev->bus_type);
+		return -EINVAL;
+	}
+
+	bus = &device_trees[dev->bus_type];
+	dev->dev_num = bus->dev_num++;
+
+	node = &bus->root.rb_node;
+	while (*node) {
+		int num = rb_entry(*node, struct device_header, node)->dev_num;
+		int result = dev->dev_num - num;
+
+		if (result < 0)
+			node = &((*node)->rb_left);
+		else if (result > 0)
+			node = &((*node)->rb_right);
+		else
+			return -EEXIST;
+	}
+
+	rb_link_node(&dev->node, parent, node);
+	rb_insert_color(&dev->node, &bus->root);
+	return 0;
+}
+
+struct device_header *device__find_dev(enum device_bus_type bus_type, u8 dev_num)
+{
+	struct rb_node *node;
+
+	if (bus_type >= DEVICE_BUS_MAX)
+		return ERR_PTR(-EINVAL);
+
+	node = device_trees[bus_type].root.rb_node;
+	while (node) {
+		struct device_header *dev = rb_entry(node, struct device_header,
+						     node);
+		if (dev_num < dev->dev_num) {
+			node = node->rb_left;
+		} else if (dev_num > dev->dev_num) {
+			node = node->rb_right;
+		} else {
+			return dev;
+		}
+	}
+
+	return NULL;
+}
+
+struct device_header *device__first_dev(enum device_bus_type bus_type)
+{
+	struct rb_node *node;
+
+	if (bus_type >= DEVICE_BUS_MAX)
+		return NULL;
+
+	node = rb_first(&device_trees[bus_type].root);
+	return node ? rb_entry(node, struct device_header, node) : NULL;
+}
+
+struct device_header *device__next_dev(struct device_header *dev)
+{
+	struct rb_node *node = rb_next(&dev->node);
+	return node ? rb_entry(node, struct device_header, node) : NULL;
+}
diff --git a/tools/kvm/hw/pci-shmem.c b/tools/kvm/hw/pci-shmem.c
index 4161335..00e5d93 100644
--- a/tools/kvm/hw/pci-shmem.c
+++ b/tools/kvm/hw/pci-shmem.c
@@ -1,3 +1,4 @@ 
+#include "kvm/devices.h"
 #include "kvm/pci-shmem.h"
 #include "kvm/virtio-pci-dev.h"
 #include "kvm/irq.h"
@@ -30,6 +31,11 @@  static struct pci_device_header pci_shmem_pci_device = {
 	.msix.pba_offset = cpu_to_le32(0x1001),		/* Use BAR 1 */
 };
 
+static struct device_header pci_shmem_device = {
+	.bus_type	= DEVICE_BUS_PCI,
+	.data		= &pci_shmem_pci_device,
+};
+
 /* registers for the Inter-VM shared memory device */
 enum ivshmem_registers {
 	INTRMASK = 0,
@@ -346,7 +352,7 @@  int shmem_parser(const struct option *opt, const char *arg, int unset)
 
 int pci_shmem__init(struct kvm *kvm)
 {
-	u8 dev, line, pin;
+	u8 line, pin;
 	char *mem;
 	int r;
 
@@ -354,7 +360,7 @@  int pci_shmem__init(struct kvm *kvm)
 		return 0;
 
 	/* Register good old INTx */
-	r = irq__register_device(PCI_DEVICE_ID_PCI_SHMEM, &dev, &pin, &line);
+	r = irq__register_device(PCI_DEVICE_ID_PCI_SHMEM, &pin, &line);
 	if (r < 0)
 		return r;
 
@@ -384,7 +390,7 @@  int pci_shmem__init(struct kvm *kvm)
 	pci_shmem_pci_device.bar[2] = cpu_to_le32(shmem_region->phys_addr | PCI_BASE_ADDRESS_SPACE_MEMORY);
 	pci_shmem_pci_device.bar_size[2] = shmem_region->size;
 
-	pci__register(&pci_shmem_pci_device, dev);
+	device__register(&pci_shmem_device);
 
 	/* Open shared memory and plug it into the guest */
 	mem = setup_shmem(shmem_region->handle, shmem_region->size,
diff --git a/tools/kvm/hw/vesa.c b/tools/kvm/hw/vesa.c
index a211491..33a675f 100644
--- a/tools/kvm/hw/vesa.c
+++ b/tools/kvm/hw/vesa.c
@@ -1,5 +1,6 @@ 
 #include "kvm/vesa.h"
 
+#include "kvm/devices.h"
 #include "kvm/virtio-pci-dev.h"
 #include "kvm/framebuffer.h"
 #include "kvm/kvm-cpu.h"
@@ -44,19 +45,24 @@  static struct pci_device_header vesa_pci_device = {
 	.bar_size[1]		= VESA_MEM_SIZE,
 };
 
+static struct device_header vesa_device = {
+	.bus_type	= DEVICE_BUS_PCI,
+	.data		= &vesa_pci_device,
+};
+
 static struct framebuffer vesafb;
 
 struct framebuffer *vesa__init(struct kvm *kvm)
 {
 	u16 vesa_base_addr;
-	u8 dev, line, pin;
+	u8 line, pin;
 	char *mem;
 	int r;
 
 	if (!kvm->cfg.vnc && !kvm->cfg.sdl)
 		return NULL;
 
-	r = irq__register_device(PCI_DEVICE_ID_VESA, &dev, &pin, &line);
+	r = irq__register_device(PCI_DEVICE_ID_VESA, &pin, &line);
 	if (r < 0)
 		return ERR_PTR(r);
 
@@ -68,7 +74,7 @@  struct framebuffer *vesa__init(struct kvm *kvm)
 	vesa_pci_device.irq_line	= line;
 	vesa_base_addr			= (u16)r;
 	vesa_pci_device.bar[0]		= cpu_to_le32(vesa_base_addr | PCI_BASE_ADDRESS_SPACE_IO);
-	pci__register(&vesa_pci_device, dev);
+	device__register(&vesa_device);
 
 	mem = mmap(NULL, VESA_MEM_SIZE, PROT_RW, MAP_ANON_NORESERVE, -1, 0);
 	if (mem == MAP_FAILED)
diff --git a/tools/kvm/include/kvm/devices.h b/tools/kvm/include/kvm/devices.h
new file mode 100644
index 0000000..c5de3de
--- /dev/null
+++ b/tools/kvm/include/kvm/devices.h
@@ -0,0 +1,27 @@ 
+#ifndef KVM__DEVICES_H
+#define KVM__DEVICES_H
+
+#include <linux/rbtree.h>
+#include <linux/types.h>
+
+enum device_bus_type {
+	DEVICE_BUS_PCI,
+	DEVICE_BUS_MMIO,
+	DEVICE_BUS_MAX,
+};
+
+struct device_header {
+	enum device_bus_type	bus_type;
+	void			*data;
+	int			dev_num;
+	struct rb_node		node;
+};
+
+int device__register(struct device_header *dev);
+struct device_header *device__find_dev(enum device_bus_type bus_type,
+				       u8 dev_num);
+
+struct device_header *device__first_dev(enum device_bus_type bus_type);
+struct device_header *device__next_dev(struct device_header *dev);
+
+#endif /* KVM__DEVICES_H */
diff --git a/tools/kvm/include/kvm/irq.h b/tools/kvm/include/kvm/irq.h
index 43fecaf..5c1274b 100644
--- a/tools/kvm/include/kvm/irq.h
+++ b/tools/kvm/include/kvm/irq.h
@@ -22,7 +22,7 @@  struct pci_dev {
 	struct list_head	lines;
 };
 
-int irq__register_device(u32 dev, u8 *num, u8 *pin, u8 *line);
+int irq__register_device(u32 dev, u8 *pin, u8 *line);
 
 struct rb_node *irq__get_pci_tree(void);
 
diff --git a/tools/kvm/include/kvm/pci.h b/tools/kvm/include/kvm/pci.h
index 26639b5..3da3811 100644
--- a/tools/kvm/include/kvm/pci.h
+++ b/tools/kvm/include/kvm/pci.h
@@ -9,7 +9,6 @@ 
 #include "kvm/kvm.h"
 #include "kvm/msi.h"
 
-#define PCI_MAX_DEVICES		256
 /*
  * PCI Configuration Mechanism #1 I/O ports. See Section 3.7.4.1.
  * ("Configuration Mechanism #1") of the PCI Local Bus Specification 2.1 for
@@ -86,7 +85,6 @@  struct pci_device_header {
 
 int pci__init(struct kvm *kvm);
 int pci__exit(struct kvm *kvm);
-int pci__register(struct pci_device_header *dev, u8 dev_num);
 struct pci_device_header *pci__find_dev(u8 dev_num);
 u32 pci_get_io_space_block(u32 size);
 void pci__config_wr(struct kvm *kvm, union pci_config_address addr, void *data, int size);
diff --git a/tools/kvm/include/kvm/virtio-mmio.h b/tools/kvm/include/kvm/virtio-mmio.h
index e0ede3c..983c8fc 100644
--- a/tools/kvm/include/kvm/virtio-mmio.h
+++ b/tools/kvm/include/kvm/virtio-mmio.h
@@ -47,6 +47,7 @@  struct virtio_mmio {
 	struct kvm		*kvm;
 	u8			irq;
 	struct virtio_mmio_hdr	hdr;
+	struct device_header	dev_hdr;
 	struct virtio_mmio_ioevent_param ioeventfds[VIRTIO_MMIO_MAX_VQ];
 };
 
diff --git a/tools/kvm/include/kvm/virtio-pci.h b/tools/kvm/include/kvm/virtio-pci.h
index 44130e0c..6d9a558 100644
--- a/tools/kvm/include/kvm/virtio-pci.h
+++ b/tools/kvm/include/kvm/virtio-pci.h
@@ -1,6 +1,7 @@ 
 #ifndef KVM__VIRTIO_PCI_H
 #define KVM__VIRTIO_PCI_H
 
+#include "kvm/devices.h"
 #include "kvm/pci.h"
 
 #include <linux/types.h>
@@ -19,6 +20,7 @@  struct virtio_pci_ioevent_param {
 
 struct virtio_pci {
 	struct pci_device_header pci_hdr;
+	struct device_header	dev_hdr;
 	void			*dev;
 
 	u16			base_addr;
diff --git a/tools/kvm/pci.c b/tools/kvm/pci.c
index c77d3cc..8d3732d 100644
--- a/tools/kvm/pci.c
+++ b/tools/kvm/pci.c
@@ -1,3 +1,4 @@ 
+#include "kvm/devices.h"
 #include "kvm/pci.h"
 #include "kvm/ioport.h"
 #include "kvm/util.h"
@@ -8,8 +9,6 @@ 
 
 #define PCI_BAR_OFFSET(b)		(offsetof(struct pci_device_header, bar[b]))
 
-static struct pci_device_header		*pci_devices[PCI_MAX_DEVICES];
-
 static union pci_config_address		pci_config_address;
 
 /* This is within our PCI gap - in an unused area.
@@ -63,20 +62,13 @@  static struct ioport_operations pci_config_address_ops = {
 
 static bool pci_device_exists(u8 bus_number, u8 device_number, u8 function_number)
 {
-	struct pci_device_header *dev;
-
 	if (pci_config_address.bus_number != bus_number)
 		return false;
 
 	if (pci_config_address.function_number != function_number)
 		return false;
 
-	if (device_number >= PCI_MAX_DEVICES)
-		return false;
-
-	dev = pci_devices[device_number];
-
-	return dev != NULL;
+	return !IS_ERR_OR_NULL(device__find_dev(DEVICE_BUS_PCI, device_number));
 }
 
 static bool pci_config_data_out(struct ioport *ioport, struct kvm *kvm, u16 port, void *data, int size)
@@ -121,12 +113,13 @@  void pci__config_wr(struct kvm *kvm, union pci_config_address addr, void *data,
 
 		offset = addr.w & 0xff;
 		if (offset < sizeof(struct pci_device_header)) {
-			void *p = pci_devices[dev_num];
+			void *p = device__find_dev(DEVICE_BUS_PCI, dev_num)->data;
+			struct pci_device_header *hdr = p;
 			u8 bar = (offset - PCI_BAR_OFFSET(0)) / (sizeof(u32));
 			u32 sz = PCI_IO_SIZE;
 
-			if (bar < 6 && pci_devices[dev_num]->bar_size[bar])
-				sz = pci_devices[dev_num]->bar_size[bar];
+			if (bar < 6 && hdr->bar_size[bar])
+				sz = hdr->bar_size[bar];
 
 			/*
 			 * If the kernel masks the BAR it would expect to find the
@@ -158,7 +151,7 @@  void pci__config_rd(struct kvm *kvm, union pci_config_address addr, void *data,
 
 		offset = addr.w & 0xff;
 		if (offset < sizeof(struct pci_device_header)) {
-			void *p = pci_devices[dev_num];
+			void *p = device__find_dev(DEVICE_BUS_PCI, dev_num)->data;
 
 			memcpy(data, p + offset, size);
 		} else {
@@ -169,22 +162,14 @@  void pci__config_rd(struct kvm *kvm, union pci_config_address addr, void *data,
 	}
 }
 
-int pci__register(struct pci_device_header *dev, u8 dev_num)
-{
-	if (dev_num >= PCI_MAX_DEVICES)
-		return -ENOSPC;
-
-	pci_devices[dev_num] = dev;
-
-	return 0;
-}
-
 struct pci_device_header *pci__find_dev(u8 dev_num)
 {
-	if (dev_num >= PCI_MAX_DEVICES)
-		return ERR_PTR(-EOVERFLOW);
+	struct device_header *hdr = device__find_dev(DEVICE_BUS_PCI, dev_num);
+
+	if (IS_ERR_OR_NULL(hdr))
+		return NULL;
 
-	return pci_devices[dev_num];
+	return hdr->data;
 }
 
 int pci__init(struct kvm *kvm)
diff --git a/tools/kvm/powerpc/irq.c b/tools/kvm/powerpc/irq.c
index e89fa3b..ae9da50 100644
--- a/tools/kvm/powerpc/irq.c
+++ b/tools/kvm/powerpc/irq.c
@@ -8,6 +8,7 @@ 
  * by the Free Software Foundation.
  */
 
+#include "kvm/devices.h"
 #include "kvm/irq.h"
 #include "kvm/kvm.h"
 #include "kvm/util.h"
@@ -31,15 +32,8 @@ 
  * generic & cope with multiple PPC platform types.
  */
 
-static int pci_devs = 0;
-
-int irq__register_device(u32 dev, u8 *num, u8 *pin, u8 *line)
+int irq__register_device(u32 dev, u8 *pin, u8 *line)
 {
-	if (pci_devs >= PCI_MAX_DEVICES)
-		die("Hit PCI device limit!\n");
-
-	*num = pci_devs++;
-
 	*pin = 1;
 	/*
 	 * Have I said how nasty I find this?  Line should be dontcare... PHB
diff --git a/tools/kvm/powerpc/spapr_pci.c b/tools/kvm/powerpc/spapr_pci.c
index 9f0ce7a..5f4016c 100644
--- a/tools/kvm/powerpc/spapr_pci.c
+++ b/tools/kvm/powerpc/spapr_pci.c
@@ -302,7 +302,7 @@  int spapr_populate_pci_devices(struct kvm *kvm,
 	/* Populate PCI devices and allocate IRQs */
 	devices = 0;
 
-	for (devid = 0; devid < PCI_MAX_DEVICES; devid++) {
+	for (devid = 0; devid < KVM_MAX_DEVICES; devid++) {
 		uint32_t *irqmap = interrupt_map[devices];
 		struct pci_device_header *hdr = pci__find_dev(devid);
 
diff --git a/tools/kvm/virtio/mmio.c b/tools/kvm/virtio/mmio.c
index fe42e95..f94c54f 100644
--- a/tools/kvm/virtio/mmio.c
+++ b/tools/kvm/virtio/mmio.c
@@ -1,3 +1,4 @@ 
+#include "kvm/devices.h"
 #include "kvm/virtio-mmio.h"
 #include "kvm/ioeventfd.h"
 #include "kvm/ioport.h"
@@ -218,7 +219,7 @@  int virtio_mmio_init(struct kvm *kvm, void *dev, struct virtio_device *vdev,
 		     int device_id, int subsys_id, int class)
 {
 	struct virtio_mmio *vmmio = vdev->virtio;
-	u8 device, pin, line;
+	u8 pin, line;
 
 	vmmio->addr	= virtio_mmio_get_io_space_block(VIRTIO_MMIO_IO_SIZE);
 	vmmio->kvm	= kvm;
@@ -235,9 +236,15 @@  int virtio_mmio_init(struct kvm *kvm, void *dev, struct virtio_device *vdev,
 		.queue_num_max	= 256,
 	};
 
-	if (irq__register_device(subsys_id, &device, &pin, &line) < 0)
+	if (irq__register_device(subsys_id, &pin, &line) < 0)
 		return -1;
 	vmmio->irq = line;
+	vmmio->dev_hdr = (struct device_header) {
+		.bus_type	= DEVICE_BUS_MMIO,
+		.data		= vmmio,
+	};
+
+	device__register(&vmmio->dev_hdr);
 
 	/*
 	 * Instantiate guest virtio-mmio devices using kernel command line
diff --git a/tools/kvm/virtio/pci.c b/tools/kvm/virtio/pci.c
index adc8efc..3d3e599 100644
--- a/tools/kvm/virtio/pci.c
+++ b/tools/kvm/virtio/pci.c
@@ -306,7 +306,7 @@  int virtio_pci__init(struct kvm *kvm, void *dev, struct virtio_device *vdev,
 		     int device_id, int subsys_id, int class)
 {
 	struct virtio_pci *vpci = vdev->virtio;
-	u8 pin, line, ndev;
+	u8 pin, line;
 	int r;
 
 	vpci->dev = dev;
@@ -343,6 +343,11 @@  int virtio_pci__init(struct kvm *kvm, void *dev, struct virtio_device *vdev,
 		.bar_size[3]		= PCI_IO_SIZE,
 	};
 
+	vpci->dev_hdr = (struct device_header) {
+		.bus_type		= DEVICE_BUS_PCI,
+		.data			= &vpci->pci_hdr,
+	};
+
 	vpci->pci_hdr.msix.cap = PCI_CAP_ID_MSIX;
 	vpci->pci_hdr.msix.next = 0;
 	/*
@@ -366,7 +371,7 @@  int virtio_pci__init(struct kvm *kvm, void *dev, struct virtio_device *vdev,
 	vpci->pci_hdr.msix.pba_offset = cpu_to_le32(1 | PCI_IO_SIZE); /* Use BAR 3 */
 	vpci->config_vector = 0;
 
-	r = irq__register_device(subsys_id, &ndev, &pin, &line);
+	r = irq__register_device(subsys_id, &pin, &line);
 	if (r < 0)
 		goto free_mmio;
 
@@ -375,7 +380,7 @@  int virtio_pci__init(struct kvm *kvm, void *dev, struct virtio_device *vdev,
 
 	vpci->pci_hdr.irq_pin	= pin;
 	vpci->pci_hdr.irq_line	= line;
-	r = pci__register(&vpci->pci_hdr, ndev);
+	r = device__register(&vpci->dev_hdr);
 	if (r < 0)
 		goto free_ioport;
 
diff --git a/tools/kvm/x86/irq.c b/tools/kvm/x86/irq.c
index 8dc90bf..7447c6b 100644
--- a/tools/kvm/x86/irq.c
+++ b/tools/kvm/x86/irq.c
@@ -17,7 +17,6 @@ 
 #define IRQCHIP_IOAPIC			2
 
 static u8		next_line	= 5;
-static u8		next_dev	= 1;
 static struct rb_root	pci_tree	= RB_ROOT;
 
 /* First 24 GSIs are routed between IRQCHIPs and IOAPICs */
@@ -86,7 +85,7 @@  static int insert(struct rb_root *root, struct pci_dev *data)
 	return 0;
 }
 
-int irq__register_device(u32 dev, u8 *num, u8 *pin, u8 *line)
+int irq__register_device(u32 dev, u8 *pin, u8 *line)
 {
 	struct pci_dev *node;
 	int r;
@@ -128,7 +127,6 @@  int irq__register_device(u32 dev, u8 *num, u8 *pin, u8 *line)
 		new->line	= next_line++;
 		*line		= new->line;
 		*pin		= node->pin;
-		*num		= next_dev++;
 
 		list_add(&new->node, &node->lines);