From 11f9628ceff019259ff12ce469deafbf50eb3075 Mon Sep 17 00:00:00 2001
From: Keqian Zhu <zhukeqian1@huawei.com>
Date: Fri, 10 Apr 2020 14:20:59 +0800
Subject: [PATCH] arm/virt: Start up CPU hot-plug

All the CPU hotplug facilities are ready. Assemble them
to start up CPU hot-plug capability for arm/virt.

Signed-off-by: Keqian Zhu <zhukeqian1@huawei.com>
Signed-off-by: Salil Mehta <salil.mehta@huawei.com>
---
 hw/arm/virt.c         | 61 ++++++++++++++++++++++++++++++++++++++++---
 include/hw/arm/virt.h |  1 +
 qom/cpu.c             |  5 ++++
 target/arm/cpu.c      |  2 ++
 4 files changed, 65 insertions(+), 4 deletions(-)

diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index c6a99e683a..112a6ae7cb 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -48,6 +48,8 @@
 #include "sysemu/cpus.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
+#include "sysemu/cpus.h"
+#include "sysemu/hw_accel.h"
 #include "hw/loader.h"
 #include "exec/address-spaces.h"
 #include "qemu/bitops.h"
@@ -649,9 +651,9 @@ static inline DeviceState *create_acpi_ged(VirtMachineState *vms)
         event |= ACPI_GED_MEM_HOTPLUG_EVT;
     }

-    /* event |= ACPI_GED_CPU_HOTPLUG_EVT;
-     * Currently CPU hotplug is not enabled.
-     */
+    if (vms->cpu_hotplug_enabled) {
+        event |= ACPI_GED_CPU_HOTPLUG_EVT;
+    }

     dev = qdev_create(NULL, TYPE_ACPI_GED);
     qdev_prop_set_uint32(dev, "ged-event", event);
@@ -2214,12 +2216,62 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev,
         object_property_set_link(cpuobj, OBJECT(secure_sysmem),
                                  "secure-memory", &error_abort);
     }
+
+    /* If we use KVM accel, we should pause all vcpus to
+     * allow hot access of vcpu registers.
+     */
+    if (dev->hotplugged && kvm_enabled()) {
+        pause_all_vcpus();
+    }
 }

 static void virt_cpu_plug(HotplugHandler *hotplug_dev,
                           DeviceState *dev, Error **errp)
 {
-    /* Currently nothing to do */
+    CPUArchId *cpu_slot;
+    CPUState *cs = CPU(dev);
+    int ncpu = cs->cpu_index;
+    MachineState *ms = MACHINE(hotplug_dev);
+    VirtMachineState *vms = VIRT_MACHINE(hotplug_dev);
+    GICv3State *gicv3;
+    ARMGICv3CommonClass *agcc;
+    Error *local_err = NULL;
+
+    if (dev->hotplugged) {
+        /* Realize GIC related parts of CPU */
+        assert(vms->gic_version == 3);
+        gicv3 = ARM_GICV3_COMMON(vms->gic);
+        agcc = ARM_GICV3_COMMON_GET_CLASS(gicv3);
+        agcc->cpu_hotplug_realize(gicv3, ncpu);
+        connect_gic_cpu_irqs(vms, ncpu);
+
+        /* Register CPU reset and trigger it manually */
+        cpu_synchronize_state(cs);
+        cpu_hotplug_register_reset(ncpu);
+        cpu_hotplug_reset_manually(ncpu);
+        cpu_synchronize_post_reset(cs);
+
+        if (kvm_enabled()) {
+            resume_all_vcpus();
+        }
+    }
+
+    if (vms->acpi_dev) {
+        hotplug_handler_plug(HOTPLUG_HANDLER(vms->acpi_dev), dev, &local_err);
+        if (local_err) {
+            goto out;
+        }
+    }
+
+    vms->boot_cpus++;
+    if (vms->fw_cfg) {
+        fw_cfg_modify_i16(vms->fw_cfg, FW_CFG_NB_CPUS, vms->boot_cpus);
+    }
+
+    cpu_slot = &ms->possible_cpus->cpus[ncpu];
+    cpu_slot->cpu = OBJECT(dev);
+out:
+    error_propagate(errp, local_err);
 }

 static void virt_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev,
@@ -2324,6 +2376,7 @@ static void virt_machine_class_init(ObjectClass *oc, void *data)
     mc->default_cpu_type = ARM_CPU_TYPE_NAME("cortex-a15");
     mc->get_default_cpu_node_id = virt_get_default_cpu_node_id;
     mc->kvm_type = virt_kvm_type;
+    mc->has_hotpluggable_cpus = true;
     assert(!mc->get_hotplug_handler);
     mc->get_hotplug_handler = virt_machine_get_hotplug_handler;
     hc->pre_plug = virt_machine_device_pre_plug_cb;
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
index b4c53d920e..a9429bed25 100644
--- a/include/hw/arm/virt.h
+++ b/include/hw/arm/virt.h
@@ -140,6 +140,7 @@ typedef struct {
     uint32_t msi_phandle;
     uint32_t iommu_phandle;
     int psci_conduit;
+    uint32_t boot_cpus;
     hwaddr highest_gpa;
     DeviceState *gic;
     DeviceState *acpi_dev;
diff --git a/qom/cpu.c b/qom/cpu.c
index f376f782d8..58cd9d5bbc 100644
--- a/qom/cpu.c
+++ b/qom/cpu.c
@@ -342,7 +342,12 @@ static void cpu_common_realizefn(DeviceState *dev, Error **errp)

     if (dev->hotplugged) {
         cpu_synchronize_post_init(cpu);
+
+#ifdef __aarch64__
+    if (!kvm_enabled())
+#endif
         cpu_resume(cpu);
+
     }

     /* NOTE: latest generic point where the cpu is fully realized */
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index 91f1e36cd8..811e5c6365 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -2598,6 +2598,8 @@ static void arm_cpu_class_init(ObjectClass *oc, void *data)
     acc->parent_reset = cc->reset;
     cc->reset = arm_cpu_reset;

+    dc->user_creatable = true;
+
     cc->class_by_name = arm_cpu_class_by_name;
     cc->has_work = arm_cpu_has_work;
     cc->cpu_exec_interrupt = arm_cpu_exec_interrupt;
--
2.19.1