| Message ID | 20240613233639.202896-19-salil.mehta@huawei.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | Support of Virtual CPU Hotplug for ARMv8 Arch | expand |
On 6/14/24 9:36 AM, Salil Mehta wrote: > Add CPU hot-unplug hooks and update hotplug hooks with additional sanity checks > for use in hotplug paths. > > Note: The functional contents of the hooks (currently left with TODO comments) > will be gradually filled in subsequent patches in an incremental approach to > patch and logic building, which would roughly include the following: > > 1. (Un)wiring of interrupts between vCPU<->GIC. > 2. Sending events to the guest for hot-(un)plug so that the guest can take > appropriate actions. > 3. Notifying the GIC about the hot-(un)plug action so that the vCPU can be > (un)stitched to the GIC CPU interface. > 4. Updating the guest with next boot information for this vCPU in the firmware. > > Co-developed-by: Keqian Zhu <zhukeqian1@huawei.com> > Signed-off-by: Keqian Zhu <zhukeqian1@huawei.com> > Signed-off-by: Salil Mehta <salil.mehta@huawei.com> > --- > hw/arm/virt.c | 105 ++++++++++++++++++++++++++++++++++++++++++++++++++ > 1 file changed, 105 insertions(+) > > diff --git a/hw/arm/virt.c b/hw/arm/virt.c > index a72cd3b20d..f6b8c21f26 100644 > --- a/hw/arm/virt.c > +++ b/hw/arm/virt.c > @@ -85,6 +85,7 @@ > #include "hw/virtio/virtio-iommu.h" > #include "hw/char/pl011.h" > #include "qemu/guest-random.h" > +#include "qapi/qmp/qdict.h" > > static GlobalProperty arm_virt_compat[] = { > { TYPE_VIRTIO_IOMMU_PCI, "aw-bits", "48" }, > @@ -3002,11 +3003,23 @@ static void virt_memory_plug(HotplugHandler *hotplug_dev, > static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > Error **errp) > { > + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); > MachineState *ms = MACHINE(hotplug_dev); > + MachineClass *mc = MACHINE_GET_CLASS(ms); > ARMCPU *cpu = ARM_CPU(dev); > CPUState *cs = CPU(dev); > CPUArchId *cpu_slot; > > + if (dev->hotplugged && !vms->acpi_dev) { > + error_setg(errp, "GED acpi device does not exists"); > + return; > + } > + > + if (dev->hotplugged && !mc->has_hotpluggable_cpus) { > + error_setg(errp, "CPU hotplug not supported on this machine"); > + return; > + } > + > /* sanity check the cpu */ > if (!object_dynamic_cast(OBJECT(cpu), ms->cpu_type)) { > error_setg(errp, "Invalid CPU type, expected cpu type: '%s'", > @@ -3049,6 +3062,22 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > } > virt_cpu_set_properties(OBJECT(cs), cpu_slot, errp); > > + /* > + * Fix the GIC for this new vCPU being plugged. The QOM CPU object for the > + * new vCPU need to be updated in the corresponding QOM GICv3CPUState object > + * We also need to re-wire the IRQs for this new CPU object. This update > + * is limited to the QOM only and does not affects the KVM. Later has > + * already been pre-sized with possible CPU at VM init time. This is a > + * workaround to the constraints posed by ARM architecture w.r.t supporting > + * CPU Hotplug. Specification does not exist for the later. > + * This patch-up is required both for {cold,hot}-plugged vCPUs. Cold-inited > + * vCPUs have their GIC state initialized during machvit_init(). > + */ > + if (vms->acpi_dev) { > + /* TODO: update GIC about this hotplug change here */ > + /* TODO: wire the GIC<->CPU irqs */ > + } > + > /* > * To give persistent presence view of vCPUs to the guest, ACPI might need > * to fake the presence of the vCPUs to the guest but keep them disabled. > @@ -3060,6 +3089,7 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > static void virt_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > Error **errp) > { > + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); > MachineState *ms = MACHINE(hotplug_dev); > CPUState *cs = CPU(dev); > CPUArchId *cpu_slot; > @@ -3068,10 +3098,81 @@ static void virt_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > cpu_slot = virt_find_cpu_slot(ms, cs->cpu_index); > cpu_slot->cpu = CPU(dev); > > + /* > + * Update the ACPI Hotplug state both for vCPUs being {hot,cold}-plugged. > + * vCPUs can be cold-plugged using '-device' option. For vCPUs being hot > + * plugged, guest is also notified. > + */ > + if (vms->acpi_dev) { > + /* TODO: update acpi hotplug state. Send cpu hotplug event to guest */ > + /* TODO: register cpu for reset & update F/W info for the next boot */ > + } > + > cs->disabled = false; > return; > } > > +static void virt_cpu_unplug_request(HotplugHandler *hotplug_dev, > + DeviceState *dev, Error **errp) > +{ > + MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine()); > + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); > + ARMCPU *cpu = ARM_CPU(dev); > + CPUState *cs = CPU(dev); > + > + if (!vms->acpi_dev || !dev->realized) { > + error_setg(errp, "GED does not exists or device is not realized!"); > + return; > + } > + How can a vCPU be unplugged even when it hasn't been realized? :) > + if (!mc->has_hotpluggable_cpus) { > + error_setg(errp, "CPU hot(un)plug not supported on this machine"); > + return; > + } > + > + if (cs->cpu_index == first_cpu->cpu_index) { > + error_setg(errp, "Boot CPU(id%d=%d:%d:%d:%d) hot-unplug not supported", > + first_cpu->cpu_index, cpu->socket_id, cpu->cluster_id, > + cpu->core_id, cpu->thread_id); > + return; > + } > + > + /* TODO: request cpu hotplug from guest */ > + > + return; > +} > + > +static void virt_cpu_unplug(HotplugHandler *hotplug_dev, DeviceState *dev, > + Error **errp) > +{ > + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); > + MachineState *ms = MACHINE(hotplug_dev); > + CPUState *cs = CPU(dev); > + CPUArchId *cpu_slot; > + > + if (!vms->acpi_dev || !dev->realized) { > + error_setg(errp, "GED does not exists or device is not realized!"); > + return; > + } > + Same question as above. > + cpu_slot = virt_find_cpu_slot(ms, cs->cpu_index); > + > + /* TODO: update the acpi cpu hotplug state for cpu hot-unplug */ > + > + /* TODO: unwire the gic-cpu irqs here */ > + /* TODO: update the GIC about this hot unplug change */ > + > + /* TODO: unregister cpu for reset & update F/W info for the next boot */ > + > + qobject_unref(dev->opts); > + dev->opts = NULL; > + > + cpu_slot->cpu = NULL; > + cs->disabled = true; > + > + return; > +} > + The 'return' isn't needed. > static void virt_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev, > DeviceState *dev, Error **errp) > { > @@ -3196,6 +3297,8 @@ static void virt_machine_device_unplug_request_cb(HotplugHandler *hotplug_dev, > } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) { > virtio_md_pci_unplug_request(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev), > errp); > + } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { > + virt_cpu_unplug_request(hotplug_dev, dev, errp); > } else { > error_setg(errp, "device unplug request for unsupported device" > " type: %s", object_get_typename(OBJECT(dev))); > @@ -3209,6 +3312,8 @@ static void virt_machine_device_unplug_cb(HotplugHandler *hotplug_dev, > virt_dimm_unplug(hotplug_dev, dev, errp); > } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) { > virtio_md_pci_unplug(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev), errp); > + } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { > + virt_cpu_unplug(hotplug_dev, dev, errp); > } else { > error_setg(errp, "virt: device unplug for unsupported device" > " type: %s", object_get_typename(OBJECT(dev))); Thanks, Gavin
Hi Gavin, > From: Gavin Shan <gshan@redhat.com> > Sent: Tuesday, August 13, 2024 2:21 AM > To: Salil Mehta <salil.mehta@huawei.com>; qemu-devel@nongnu.org; > qemu-arm@nongnu.org; mst@redhat.com > > On 6/14/24 9:36 AM, Salil Mehta wrote: > > Add CPU hot-unplug hooks and update hotplug hooks with additional > > sanity checks for use in hotplug paths. > > > > Note: The functional contents of the hooks (currently left with TODO > > comments) will be gradually filled in subsequent patches in an > > incremental approach to patch and logic building, which would roughly > include the following: > > > > 1. (Un)wiring of interrupts between vCPU<->GIC. > > 2. Sending events to the guest for hot-(un)plug so that the guest can take > > appropriate actions. > > 3. Notifying the GIC about the hot-(un)plug action so that the vCPU can be > > (un)stitched to the GIC CPU interface. > > 4. Updating the guest with next boot information for this vCPU in the > firmware. > > > > Co-developed-by: Keqian Zhu <zhukeqian1@huawei.com> > > Signed-off-by: Keqian Zhu <zhukeqian1@huawei.com> > > Signed-off-by: Salil Mehta <salil.mehta@huawei.com> > > --- > > hw/arm/virt.c | 105 > ++++++++++++++++++++++++++++++++++++++++++++++++++ > > 1 file changed, 105 insertions(+) > > > > diff --git a/hw/arm/virt.c b/hw/arm/virt.c index > > a72cd3b20d..f6b8c21f26 100644 > > --- a/hw/arm/virt.c > > +++ b/hw/arm/virt.c > > @@ -85,6 +85,7 @@ > > #include "hw/virtio/virtio-iommu.h" > > #include "hw/char/pl011.h" > > #include "qemu/guest-random.h" > > +#include "qapi/qmp/qdict.h" > > > > static GlobalProperty arm_virt_compat[] = { > > { TYPE_VIRTIO_IOMMU_PCI, "aw-bits", "48" }, @@ -3002,11 +3003,23 > > @@ static void virt_memory_plug(HotplugHandler *hotplug_dev, > > static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState > *dev, > > Error **errp) > > { > > + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); > > MachineState *ms = MACHINE(hotplug_dev); > > + MachineClass *mc = MACHINE_GET_CLASS(ms); > > ARMCPU *cpu = ARM_CPU(dev); > > CPUState *cs = CPU(dev); > > CPUArchId *cpu_slot; > > > > + if (dev->hotplugged && !vms->acpi_dev) { > > + error_setg(errp, "GED acpi device does not exists"); > > + return; > > + } > > + > > + if (dev->hotplugged && !mc->has_hotpluggable_cpus) { > > + error_setg(errp, "CPU hotplug not supported on this machine"); > > + return; > > + } > > + > > /* sanity check the cpu */ > > if (!object_dynamic_cast(OBJECT(cpu), ms->cpu_type)) { > > error_setg(errp, "Invalid CPU type, expected cpu type: > > '%s'", @@ -3049,6 +3062,22 @@ static void > virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > > } > > virt_cpu_set_properties(OBJECT(cs), cpu_slot, errp); > > > > + /* > > + * Fix the GIC for this new vCPU being plugged. The QOM CPU object for the > > + * new vCPU need to be updated in the corresponding QOM GICv3CPUState object > > + * We also need to re-wire the IRQs for this new CPU object. This update > > + * is limited to the QOM only and does not affects the KVM. Later has > > + * already been pre-sized with possible CPU at VM init time. This is a > > + * workaround to the constraints posed by ARM architecture w.r.t supporting > > + * CPU Hotplug. Specification does not exist for the later. > > + * This patch-up is required both for {cold,hot}-plugged vCPUs. Cold-inited > > + * vCPUs have their GIC state initialized during machvit_init(). > > + */ > > + if (vms->acpi_dev) { > > + /* TODO: update GIC about this hotplug change here */ > > + /* TODO: wire the GIC<->CPU irqs */ > > + } > > + > > /* > > * To give persistent presence view of vCPUs to the guest, ACPI might need > > * to fake the presence of the vCPUs to the guest but keep them disabled. > > @@ -3060,6 +3089,7 @@ static void virt_cpu_pre_plug(HotplugHandler > *hotplug_dev, DeviceState *dev, > > static void virt_cpu_plug(HotplugHandler *hotplug_dev, DeviceState > *dev, > > Error **errp) > > { > > + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); > > MachineState *ms = MACHINE(hotplug_dev); > > CPUState *cs = CPU(dev); > > CPUArchId *cpu_slot; > > @@ -3068,10 +3098,81 @@ static void virt_cpu_plug(HotplugHandler > *hotplug_dev, DeviceState *dev, > > cpu_slot = virt_find_cpu_slot(ms, cs->cpu_index); > > cpu_slot->cpu = CPU(dev); > > > > + /* > > + * Update the ACPI Hotplug state both for vCPUs being {hot,cold}- > plugged. > > + * vCPUs can be cold-plugged using '-device' option. For vCPUs being > hot > > + * plugged, guest is also notified. > > + */ > > + if (vms->acpi_dev) { > > + /* TODO: update acpi hotplug state. Send cpu hotplug event to guest > */ > > + /* TODO: register cpu for reset & update F/W info for the next boot > */ > > + } > > + > > cs->disabled = false; > > return; > > } > > > > +static void virt_cpu_unplug_request(HotplugHandler *hotplug_dev, > > + DeviceState *dev, Error **errp) { > > + MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine()); > > + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); > > + ARMCPU *cpu = ARM_CPU(dev); > > + CPUState *cs = CPU(dev); > > + > > + if (!vms->acpi_dev || !dev->realized) { > > + error_setg(errp, "GED does not exists or device is not realized!"); > > + return; > > + } > > + > > How can a vCPU be unplugged even when it hasn't been realized? :)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c index a72cd3b20d..f6b8c21f26 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -85,6 +85,7 @@ #include "hw/virtio/virtio-iommu.h" #include "hw/char/pl011.h" #include "qemu/guest-random.h" +#include "qapi/qmp/qdict.h" static GlobalProperty arm_virt_compat[] = { { TYPE_VIRTIO_IOMMU_PCI, "aw-bits", "48" }, @@ -3002,11 +3003,23 @@ static void virt_memory_plug(HotplugHandler *hotplug_dev, static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp) { + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); MachineState *ms = MACHINE(hotplug_dev); + MachineClass *mc = MACHINE_GET_CLASS(ms); ARMCPU *cpu = ARM_CPU(dev); CPUState *cs = CPU(dev); CPUArchId *cpu_slot; + if (dev->hotplugged && !vms->acpi_dev) { + error_setg(errp, "GED acpi device does not exists"); + return; + } + + if (dev->hotplugged && !mc->has_hotpluggable_cpus) { + error_setg(errp, "CPU hotplug not supported on this machine"); + return; + } + /* sanity check the cpu */ if (!object_dynamic_cast(OBJECT(cpu), ms->cpu_type)) { error_setg(errp, "Invalid CPU type, expected cpu type: '%s'", @@ -3049,6 +3062,22 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, } virt_cpu_set_properties(OBJECT(cs), cpu_slot, errp); + /* + * Fix the GIC for this new vCPU being plugged. The QOM CPU object for the + * new vCPU need to be updated in the corresponding QOM GICv3CPUState object + * We also need to re-wire the IRQs for this new CPU object. This update + * is limited to the QOM only and does not affects the KVM. Later has + * already been pre-sized with possible CPU at VM init time. This is a + * workaround to the constraints posed by ARM architecture w.r.t supporting + * CPU Hotplug. Specification does not exist for the later. + * This patch-up is required both for {cold,hot}-plugged vCPUs. Cold-inited + * vCPUs have their GIC state initialized during machvit_init(). + */ + if (vms->acpi_dev) { + /* TODO: update GIC about this hotplug change here */ + /* TODO: wire the GIC<->CPU irqs */ + } + /* * To give persistent presence view of vCPUs to the guest, ACPI might need * to fake the presence of the vCPUs to the guest but keep them disabled. @@ -3060,6 +3089,7 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, static void virt_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp) { + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); MachineState *ms = MACHINE(hotplug_dev); CPUState *cs = CPU(dev); CPUArchId *cpu_slot; @@ -3068,10 +3098,81 @@ static void virt_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, cpu_slot = virt_find_cpu_slot(ms, cs->cpu_index); cpu_slot->cpu = CPU(dev); + /* + * Update the ACPI Hotplug state both for vCPUs being {hot,cold}-plugged. + * vCPUs can be cold-plugged using '-device' option. For vCPUs being hot + * plugged, guest is also notified. + */ + if (vms->acpi_dev) { + /* TODO: update acpi hotplug state. Send cpu hotplug event to guest */ + /* TODO: register cpu for reset & update F/W info for the next boot */ + } + cs->disabled = false; return; } +static void virt_cpu_unplug_request(HotplugHandler *hotplug_dev, + DeviceState *dev, Error **errp) +{ + MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine()); + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); + ARMCPU *cpu = ARM_CPU(dev); + CPUState *cs = CPU(dev); + + if (!vms->acpi_dev || !dev->realized) { + error_setg(errp, "GED does not exists or device is not realized!"); + return; + } + + if (!mc->has_hotpluggable_cpus) { + error_setg(errp, "CPU hot(un)plug not supported on this machine"); + return; + } + + if (cs->cpu_index == first_cpu->cpu_index) { + error_setg(errp, "Boot CPU(id%d=%d:%d:%d:%d) hot-unplug not supported", + first_cpu->cpu_index, cpu->socket_id, cpu->cluster_id, + cpu->core_id, cpu->thread_id); + return; + } + + /* TODO: request cpu hotplug from guest */ + + return; +} + +static void virt_cpu_unplug(HotplugHandler *hotplug_dev, DeviceState *dev, + Error **errp) +{ + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); + MachineState *ms = MACHINE(hotplug_dev); + CPUState *cs = CPU(dev); + CPUArchId *cpu_slot; + + if (!vms->acpi_dev || !dev->realized) { + error_setg(errp, "GED does not exists or device is not realized!"); + return; + } + + cpu_slot = virt_find_cpu_slot(ms, cs->cpu_index); + + /* TODO: update the acpi cpu hotplug state for cpu hot-unplug */ + + /* TODO: unwire the gic-cpu irqs here */ + /* TODO: update the GIC about this hot unplug change */ + + /* TODO: unregister cpu for reset & update F/W info for the next boot */ + + qobject_unref(dev->opts); + dev->opts = NULL; + + cpu_slot->cpu = NULL; + cs->disabled = true; + + return; +} + static void virt_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp) { @@ -3196,6 +3297,8 @@ static void virt_machine_device_unplug_request_cb(HotplugHandler *hotplug_dev, } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) { virtio_md_pci_unplug_request(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev), errp); + } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { + virt_cpu_unplug_request(hotplug_dev, dev, errp); } else { error_setg(errp, "device unplug request for unsupported device" " type: %s", object_get_typename(OBJECT(dev))); @@ -3209,6 +3312,8 @@ static void virt_machine_device_unplug_cb(HotplugHandler *hotplug_dev, virt_dimm_unplug(hotplug_dev, dev, errp); } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) { virtio_md_pci_unplug(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev), errp); + } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { + virt_cpu_unplug(hotplug_dev, dev, errp); } else { error_setg(errp, "virt: device unplug for unsupported device" " type: %s", object_get_typename(OBJECT(dev)));