253 lines
7.5 KiB
Diff
253 lines
7.5 KiB
Diff
From 18b3736c7a0dd1915401981750a5d1a3b3560031 Mon Sep 17 00:00:00 2001
|
|
From: zhengchuan <zhengchuan@huawei.com>
|
|
Date: Wed, 30 Nov 2022 15:01:12 +0800
|
|
Subject: [PATCH 4/6] migration/migration-pin: add domainMigrationPid for
|
|
qemuMonitorCallbacks
|
|
|
|
add domainMigrationPid for qemuMonitorCallbacks
|
|
|
|
Signed-off-by:zhengchuan<zhengchuan@huawei.com>
|
|
---
|
|
src/qemu/qemu_process.c | 189 ++++++++++++++++++++++++++++++++++++++++
|
|
src/qemu/qemu_process.h | 13 +++
|
|
2 files changed, 202 insertions(+)
|
|
|
|
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
|
|
index 9cbf34a046..ce1c597da1 100644
|
|
--- a/src/qemu/qemu_process.c
|
|
+++ b/src/qemu/qemu_process.c
|
|
@@ -1914,6 +1914,7 @@ static qemuMonitorCallbacks monitorCallbacks = {
|
|
.domainPRManagerStatusChanged = qemuProcessHandlePRManagerStatusChanged,
|
|
.domainRdmaGidStatusChanged = qemuProcessHandleRdmaGidStatusChanged,
|
|
.domainGuestCrashloaded = qemuProcessHandleGuestCrashloaded,
|
|
+ .domainMigrationPid = qemuProcessHandleMigrationPid,
|
|
};
|
|
|
|
static void
|
|
@@ -2782,6 +2783,194 @@ qemuProcessResctrlCreate(virQEMUDriverPtr driver,
|
|
}
|
|
|
|
|
|
+int
|
|
+qemuProcessSetupMigration(virDomainObjPtr vm,
|
|
+ virDomainMigrationIDDefPtr migration)
|
|
+{
|
|
+ return qemuProcessSetupPid(vm, migration->thread_id,
|
|
+ VIR_CGROUP_THREAD_MIGRATION_THREAD,
|
|
+ 0,
|
|
+ vm->def->cputune.emulatorpin,
|
|
+ vm->def->cputune.emulator_period,
|
|
+ vm->def->cputune.emulator_quota,
|
|
+ &migration->sched);
|
|
+}
|
|
+
|
|
+
|
|
+unsigned char *
|
|
+virParseCPUList(int *cpumaplen, const char *cpulist, int maxcpu)
|
|
+{
|
|
+ unsigned char *cpumap = NULL;
|
|
+ virBitmapPtr map = NULL;
|
|
+
|
|
+ if (cpulist[0] == 'r') {
|
|
+ map = virBitmapNew(maxcpu);
|
|
+ if (!map)
|
|
+ return NULL;
|
|
+ virBitmapSetAll(map);
|
|
+ } else {
|
|
+ if (virBitmapParse(cpulist, &map, 1024) < 0 ||
|
|
+ virBitmapIsAllClear(map)) {
|
|
+ goto cleanup;
|
|
+ }
|
|
+
|
|
+ int lastcpu = virBitmapLastSetBit(map);
|
|
+ if (lastcpu >= maxcpu)
|
|
+ goto cleanup;
|
|
+ }
|
|
+
|
|
+ if (virBitmapToData(map, &cpumap, cpumaplen) < 0)
|
|
+ VIR_ERROR(_("Bitmap to data failure"));
|
|
+
|
|
+ cleanup:
|
|
+ virBitmapFree(map);
|
|
+ return cpumap;
|
|
+}
|
|
+
|
|
+
|
|
+/*
|
|
+ * If priv->pcpumap is NULL, it means migrationpin command is not called,
|
|
+ * otherwise we set the affinity of migration thread by migrationpin.
|
|
+ */
|
|
+static virBitmapPtr
|
|
+qemuProcessGetPcpumap(qemuDomainObjPrivatePtr priv)
|
|
+{
|
|
+ int cpumaplen = 0;
|
|
+ int maxcpu = 0;
|
|
+ g_autofree unsigned char *cpumap = NULL;
|
|
+ virBitmapPtr pcpumap = NULL;
|
|
+
|
|
+ if(priv->pcpumap)
|
|
+ return priv->pcpumap;
|
|
+
|
|
+ if (!(priv->migrationThreadPinList) || STREQ(priv->migrationThreadPinList, "")) {
|
|
+ VIR_ERROR(_("didn't set the migratin thread pin"));
|
|
+ return NULL;
|
|
+ }
|
|
+
|
|
+ /* judge whether migration.pin is default value or not */
|
|
+ if (STREQ(priv->migrationThreadPinList, "none"))
|
|
+ return NULL;
|
|
+
|
|
+ maxcpu = virHostCPUGetCount();
|
|
+ if (maxcpu < 0) {
|
|
+ VIR_ERROR(_("get the cpu count of host failure"));
|
|
+ return NULL;
|
|
+ }
|
|
+
|
|
+ cpumap = virParseCPUList(&cpumaplen, priv->migrationThreadPinList, maxcpu);
|
|
+ if (!cpumap) {
|
|
+ VIR_ERROR(_("parse migration.pin params failure : migration.pin = %s"),
|
|
+ priv->migrationThreadPinList);
|
|
+ return NULL;
|
|
+ }
|
|
+
|
|
+ if (!(pcpumap = virBitmapNewData(cpumap, cpumaplen))) {
|
|
+ VIR_ERROR(_("Bitmap data failure"));
|
|
+ return pcpumap;
|
|
+ }
|
|
+
|
|
+ return pcpumap;
|
|
+}
|
|
+
|
|
+
|
|
+/*
|
|
+ * In order to set migration thread affinity when vm is migrating,
|
|
+ * we should create the cgroup for migration thread.
|
|
+ */
|
|
+static void
|
|
+qemuProcessSetMigthreadAffinity(qemuDomainObjPrivatePtr priv,
|
|
+ virBitmapPtr pcpumap,
|
|
+ int mpid)
|
|
+{
|
|
+ int migration_id = 0;
|
|
+ virCgroupPtr cgroup_migthread = NULL;
|
|
+
|
|
+ if (!pcpumap)
|
|
+ return;
|
|
+
|
|
+ if (virCgroupHasController(priv->cgroup,
|
|
+ VIR_CGROUP_CONTROLLER_CPUSET)) {
|
|
+ if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD,
|
|
+ migration_id, false, &cgroup_migthread) < 0)
|
|
+ goto cleanup;
|
|
+
|
|
+ if (qemuSetupCgroupCpusetCpus(cgroup_migthread, pcpumap) < 0) {
|
|
+ virReportError(VIR_ERR_OPERATION_INVALID,
|
|
+ _("failed to set cpuset.cpus in cgroup"
|
|
+ " for migration%d thread"), migration_id);
|
|
+ goto cleanup;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ if (virProcessSetAffinity(mpid, pcpumap) < 0)
|
|
+ VIR_WARN("failed to set affinity in migration");
|
|
+
|
|
+ cleanup:
|
|
+ if (cgroup_migthread)
|
|
+ virCgroupFree(&cgroup_migthread);
|
|
+ return;
|
|
+}
|
|
+
|
|
+
|
|
+int
|
|
+qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
|
|
+ virDomainObjPtr vm,
|
|
+ int mpid,
|
|
+ void *opaque G_GNUC_UNUSED)
|
|
+{
|
|
+ qemuDomainObjPrivatePtr priv;
|
|
+ char *mpidStr = NULL;
|
|
+ virDomainMigrationIDDefPtr migration = NULL;
|
|
+ virBitmapPtr pcpumap = NULL;
|
|
+ virObjectLock(vm);
|
|
+
|
|
+ VIR_INFO("Migrating domain %p %s, migration pid %d",
|
|
+ vm, vm->def->name, mpid);
|
|
+
|
|
+ priv = vm->privateData;
|
|
+ if (priv->job.asyncJob == QEMU_ASYNC_JOB_NONE) {
|
|
+ VIR_DEBUG("got MIGRATION_PID event without a migration job");
|
|
+ goto cleanup;
|
|
+ }
|
|
+
|
|
+ if (VIR_ALLOC(migration) < 0) {
|
|
+ VIR_ERROR(_("alloc migrationIDDefPtr failure"));
|
|
+ goto cleanup;
|
|
+ }
|
|
+ migration->thread_id = mpid;
|
|
+
|
|
+ if (qemuProcessSetupMigration(vm, migration) < 0) {
|
|
+ VIR_ERROR(_("fail to setup migration cgroup"));
|
|
+ goto cleanup;
|
|
+ }
|
|
+
|
|
+ mpidStr = g_strdup_printf("%d", mpid);
|
|
+
|
|
+ VIR_FREE(priv->migrationPids);
|
|
+ priv->migrationPids = mpidStr;
|
|
+
|
|
+ pcpumap = qemuProcessGetPcpumap(priv);
|
|
+
|
|
+ if (!pcpumap)
|
|
+ goto cleanup;
|
|
+
|
|
+ qemuProcessSetMigthreadAffinity(priv, pcpumap, mpid);
|
|
+
|
|
+ cleanup:
|
|
+ /*
|
|
+ * If the value of pcpumap is setted by priv->migrationThreadPinList,
|
|
+ * we need to free pcpumap.
|
|
+ */
|
|
+ if (pcpumap != priv->pcpumap)
|
|
+ virBitmapFree(pcpumap);
|
|
+ virDomainMigrationIDDefFree(migration);
|
|
+ virObjectUnlock(vm);
|
|
+
|
|
+ return 0;
|
|
+}
|
|
+
|
|
+
|
|
static char *
|
|
qemuProcessBuildPRHelperPidfilePath(virDomainObjPtr vm)
|
|
{
|
|
diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h
|
|
index 15e67b9762..be062b9f95 100644
|
|
--- a/src/qemu/qemu_process.h
|
|
+++ b/src/qemu/qemu_process.h
|
|
@@ -25,6 +25,7 @@
|
|
#include "qemu_domain.h"
|
|
#include "virstoragefile.h"
|
|
#include "vireventthread.h"
|
|
+#include "domain_conf.h"
|
|
|
|
int qemuProcessPrepareMonitorChr(virDomainChrSourceDefPtr monConfig,
|
|
const char *domainDir);
|
|
@@ -236,3 +237,15 @@ qemuProcessQMPPtr qemuProcessQMPNew(const char *binary,
|
|
void qemuProcessQMPFree(qemuProcessQMPPtr proc);
|
|
|
|
int qemuProcessQMPStart(qemuProcessQMPPtr proc);
|
|
+
|
|
+int qemuProcessSetupMigration(virDomainObjPtr vm,
|
|
+ virDomainMigrationIDDefPtr migration);
|
|
+
|
|
+unsigned char * virParseCPUList(int *cpumaplen,
|
|
+ const char *cpulist,
|
|
+ int maxcpu);
|
|
+
|
|
+int qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
|
|
+ virDomainObjPtr vm,
|
|
+ int mpid,
|
|
+ void *opaque);
|
|
--
|
|
2.25.1
|
|
|