From eba24068980694c11fd06bfb146dd672f2c1788d Mon Sep 17 00:00:00 2001 From: Chen Qun Date: Fri, 10 Apr 2020 14:16:40 +0800 Subject: [PATCH] arm/virt: Start up CPU hot-plug and cold-plug All the CPU hotplug facilities are ready. Assemble them to start up CPU hot-plug capability for arm/virt. This also adds CPU cold plug support to arm virt machine board. CPU cold plug means adding CPU by using "-device xx-arm-cpu" when we bring up Qemu. Signed-off-by: Salil Mehta Signed-off-by: Keqian Zhu (cherry picked from commit d5d118431a64c4202cb28c27bdea0f8667beb5dd) --- ...-Start-up-CPU-hot-plug-and-cold-plug.patch | 237 ++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 arm-virt-Start-up-CPU-hot-plug-and-cold-plug.patch diff --git a/arm-virt-Start-up-CPU-hot-plug-and-cold-plug.patch b/arm-virt-Start-up-CPU-hot-plug-and-cold-plug.patch new file mode 100644 index 0000000..d4970f1 --- /dev/null +++ b/arm-virt-Start-up-CPU-hot-plug-and-cold-plug.patch @@ -0,0 +1,237 @@ +From a2d8cf86a379bb161cdae850824c9e80fb370599 Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Fri, 10 Apr 2020 14:16:40 +0800 +Subject: [PATCH] arm/virt: Start up CPU hot-plug and cold-plug + +All the CPU hotplug facilities are ready. Assemble them +to start up CPU hot-plug capability for arm/virt. + +This also adds CPU cold plug support to arm virt machine +board. CPU cold plug means adding CPU by using "-device +xx-arm-cpu" when we bring up Qemu. + +Signed-off-by: Salil Mehta +Signed-off-by: Keqian Zhu +--- + hw/arm/virt.c | 110 ++++++++++++++++++++++++++++++++++++++++-- + hw/core/cpu-common.c | 4 ++ + include/hw/arm/virt.h | 1 + + target/arm/cpu.c | 2 + + 4 files changed, 113 insertions(+), 4 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 4eb1b44729..b81d22d68f 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -52,6 +52,8 @@ + #include "sysemu/tpm.h" + #include "sysemu/kvm.h" + #include "sysemu/hvf.h" ++#include "sysemu/cpus.h" ++#include "sysemu/hw_accel.h" + #include "hw/loader.h" + #include "qapi/error.h" + #include "qemu/bitops.h" +@@ -703,9 +705,9 @@ static inline DeviceState *create_acpi_ged(VirtMachineState *vms) + event |= ACPI_GED_NVDIMM_HOTPLUG_EVT; + } + +- /* event |= ACPI_GED_CPU_HOTPLUG_EVT; +- * Currently CPU hotplug is not enabled. +- */ ++ if (vms->cpu_hotplug_enabled) { ++ event |= ACPI_GED_CPU_HOTPLUG_EVT; ++ } + + dev = qdev_new(TYPE_ACPI_GED); + qdev_prop_set_uint32(dev, "ged-event", event); +@@ -2555,11 +2557,18 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); + VirtMachineClass *vmc = VIRT_MACHINE_GET_CLASS(hotplug_dev); + const CPUArchIdList *possible_cpus = mc->possible_cpu_arch_ids(ms); ++ const CPUArchId *cpu_slot = NULL; + MemoryRegion *sysmem = get_system_memory(); + int smp_clusters = ms->smp.clusters; + int smp_cores = ms->smp.cores; + int smp_threads = ms->smp.threads; + ++ if (!object_dynamic_cast(OBJECT(cpu), ms->cpu_type)) { ++ error_setg(errp, "Invalid CPU type, expected cpu type: '%s'", ++ ms->cpu_type); ++ return; ++ } ++ + /* if cpu idx is not set, set it based on socket/cluster/core/thread + * properties + */ +@@ -2593,6 +2602,20 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, + cs->cpu_index = idx_from_topo_ids(smp_clusters, smp_cores, smp_threads, &topo); + } + ++ /* Some hotplug capability checks */ ++ if (cs->cpu_index >= ms->smp.cpus) { ++ if (!vms->acpi_dev) { ++ error_setg(errp, "CPU cold/hot plug is disabled: " ++ "missing acpi device."); ++ return; ++ } ++ if (!vms->cpu_hotplug_enabled) { ++ error_setg(errp, "CPU cold/hot plug is disabled: " ++ "should use AArch64 CPU and GICv3."); ++ return; ++ } ++ } ++ + /* if 'address' properties socket-id/cluster-id/core-id/thread-id are not + * set, set them so that machine_query_hotpluggable_cpus would show correct + * values +@@ -2631,6 +2654,13 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, + object_property_set_int(cpuobj, "mp-affinity", + possible_cpus->cpus[cs->cpu_index].arch_id, NULL); + ++ cpu_slot = &possible_cpus->cpus[cs->cpu_index]; ++ if (cpu_slot->cpu) { ++ error_setg(errp, "CPU[%d] with mp_affinity %" PRIu64 " exists", ++ cs->cpu_index, cpu->mp_affinity); ++ return; ++ } ++ + numa_cpu_pre_plug(&possible_cpus->cpus[cs->cpu_index], DEVICE(cpuobj), + &error_fatal); + +@@ -2716,12 +2746,83 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, + &error_abort); + } + } ++ ++ /* If we use KVM accel, we should pause all vcpus to ++ * allow hot access of vcpu registers. ++ */ ++ if (dev->hotplugged && kvm_enabled()) { ++ pause_all_vcpus(); ++ } + } + + static void virt_cpu_plug(HotplugHandler *hotplug_dev, + DeviceState *dev, Error **errp) + { +- /* Currently nothing to do */ ++ CPUArchId *cpu_slot; ++ CPUState *cs = CPU(dev); ++ int ncpu = cs->cpu_index; ++ MachineState *ms = MACHINE(hotplug_dev); ++ VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); ++ bool pmu = object_property_get_bool(OBJECT(first_cpu), "pmu", NULL); ++ bool steal_time = object_property_get_bool(OBJECT(first_cpu), ++ "kvm-steal-time", NULL); ++ GICv3State *gicv3; ++ ARMGICv3CommonClass *agcc; ++ Error *local_err = NULL; ++ ++ /* For CPU that is cold/hot plugged */ ++ if (ncpu >= ms->smp.cpus) { ++ /* Realize GIC related parts of CPU */ ++ assert(vms->gic_version == 3); ++ gicv3 = ARM_GICV3_COMMON(vms->gic); ++ agcc = ARM_GICV3_COMMON_GET_CLASS(gicv3); ++ agcc->cpu_hotplug_realize(gicv3, ncpu); ++ connect_gic_cpu_irqs(vms, ncpu); ++ ++ /* Init PMU and steal_time part */ ++ if (kvm_enabled()) { ++ hwaddr pvtime_reg_base = vms->memmap[VIRT_PVTIME].base; ++ ++ if (pmu) { ++ assert(arm_feature(&ARM_CPU(cs)->env, ARM_FEATURE_PMU)); ++ if (kvm_irqchip_in_kernel()) { ++ kvm_arm_pmu_set_irq(cs, PPI(VIRTUAL_PMU_IRQ)); ++ } ++ kvm_arm_pmu_init(cs); ++ } ++ if (steal_time) { ++ kvm_arm_pvtime_init(cs, pvtime_reg_base + ++ ncpu * PVTIME_SIZE_PER_CPU); ++ } ++ } ++ ++ /* Register CPU reset and trigger it manually */ ++ cpu_synchronize_state(cs); ++ cpu_hotplug_register_reset(ncpu); ++ cpu_hotplug_reset_manually(ncpu); ++ cpu_synchronize_post_reset(cs); ++ } ++ ++ if (dev->hotplugged && kvm_enabled()) { ++ resume_all_vcpus(); ++ } ++ ++ if (vms->acpi_dev) { ++ hotplug_handler_plug(HOTPLUG_HANDLER(vms->acpi_dev), dev, &local_err); ++ if (local_err) { ++ goto out; ++ } ++ } ++ ++ vms->boot_cpus++; ++ if (vms->fw_cfg) { ++ fw_cfg_modify_i16(vms->fw_cfg, FW_CFG_NB_CPUS, vms->boot_cpus); ++ } ++ ++ cpu_slot = &ms->possible_cpus->cpus[ncpu]; ++ cpu_slot->cpu = OBJECT(dev); ++out: ++ error_propagate(errp, local_err); + } + + static void virt_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev, +@@ -2940,6 +3041,7 @@ static void virt_machine_class_init(ObjectClass *oc, void *data) + mc->default_cpu_type = ARM_CPU_TYPE_NAME("cortex-a15"); + mc->get_default_cpu_node_id = virt_get_default_cpu_node_id; + mc->kvm_type = virt_kvm_type; ++ mc->has_hotpluggable_cpus = true; + assert(!mc->get_hotplug_handler); + mc->get_hotplug_handler = virt_machine_get_hotplug_handler; + hc->pre_plug = virt_machine_device_pre_plug_cb; +diff --git a/hw/core/cpu-common.c b/hw/core/cpu-common.c +index 9e3241b430..b8d1d820cb 100644 +--- a/hw/core/cpu-common.c ++++ b/hw/core/cpu-common.c +@@ -208,6 +208,10 @@ static void cpu_common_realizefn(DeviceState *dev, Error **errp) + + if (dev->hotplugged) { + cpu_synchronize_post_init(cpu); ++ ++#ifdef __aarch64__ ++ if (!kvm_enabled()) ++#endif + cpu_resume(cpu); + } + +diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h +index c371d377e0..4ddee19b18 100644 +--- a/include/hw/arm/virt.h ++++ b/include/hw/arm/virt.h +@@ -168,6 +168,7 @@ struct VirtMachineState { + uint32_t msi_phandle; + uint32_t iommu_phandle; + int psci_conduit; ++ uint32_t boot_cpus; + hwaddr highest_gpa; + DeviceState *gic; + DeviceState *acpi_dev; +diff --git a/target/arm/cpu.c b/target/arm/cpu.c +index 9fd8e57971..d550022f18 100644 +--- a/target/arm/cpu.c ++++ b/target/arm/cpu.c +@@ -2580,6 +2580,8 @@ static void arm_cpu_class_init(ObjectClass *oc, void *data) + device_class_set_props(dc, arm_cpu_properties); + device_class_set_parent_reset(dc, arm_cpu_reset, &acc->parent_reset); + ++ dc->user_creatable = true; ++ + cc->class_by_name = arm_cpu_class_by_name; + cc->has_work = arm_cpu_has_work; + cc->dump_state = arm_cpu_dump_state; +-- +2.27.0 +