From 8c07702d9280ef5235bb2bdb1ee472452cffca02 Mon Sep 17 00:00:00 2001 From: zhengchuan Date: Wed, 30 Nov 2022 16:47:30 +0800 Subject: [PATCH 6/6] migration/multifd-pin: support migration multifd thread pin support migration multifd thread pin by configuration. Signed-off-by:zhengchuan --- src/qemu/qemu_migration.c | 2 + src/qemu/qemu_process.c | 119 ++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 5 ++ 3 files changed, 126 insertions(+) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index b47e69c642..c2a694eff1 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2534,6 +2534,8 @@ qemuMigrationDstPrepareAny(virQEMUDriverPtr driver, priv = vm->privateData; priv->origname = g_strdup(origname); VIR_FREE(priv->migrationPids); + VIR_FREE(priv->migrationMultiFdPids); + priv->migrationMultiFdCount = 0; if (taint_hook) { /* Domain XML has been altered by a hook script. */ diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index ce1c597da1..0f3e651630 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -1629,6 +1629,57 @@ qemuProcessHandleSpiceMigrated(qemuMonitorPtr mon G_GNUC_UNUSED, } +static void +qemuProcessHandleMigrationPinStatus(qemuDomainObjPrivatePtr priv, int status) +{ + if (priv->job.asyncJob != QEMU_ASYNC_JOB_MIGRATION_OUT) + return; + + switch (status) { + case QEMU_MONITOR_MIGRATION_STATUS_INACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_SETUP: + case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_PRE_SWITCHOVER: + case QEMU_MONITOR_MIGRATION_STATUS_DEVICE: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY: + case QEMU_MONITOR_MIGRATION_STATUS_CANCELLING: + case QEMU_MONITOR_MIGRATION_STATUS_COMPLETED: + case QEMU_MONITOR_MIGRATION_STATUS_WAIT_UNPLUG: + break; + case QEMU_MONITOR_MIGRATION_STATUS_ERROR: + /* + * migration thread is still running, + * so we can't delete migration Cgroup. + */ + VIR_FREE(priv->migrationPids); + VIR_FREE(priv->migrationMultiFdPids); + VIR_FREE(priv->migrationThreadPinList); + priv->migrationMultiFdCount = 0; + virBitmapFree(priv->pcpumap); + priv->pcpumap = NULL; + break; + case QEMU_MONITOR_MIGRATION_STATUS_CANCELLED: + VIR_FREE(priv->migrationPids); + VIR_FREE(priv->migrationMultiFdPids); + VIR_FREE(priv->migrationThreadPinList); + priv->migrationMultiFdCount = 0; + virBitmapFree(priv->pcpumap); + priv->pcpumap = NULL; + if (virCgroupDelThread(priv->cgroup, + VIR_CGROUP_THREAD_MIGRATION_THREAD, 0) < 0) + VIR_WARN("Failed to delete migration thread Cgroup!"); + VIR_INFO("success to free pcpumap and migrationPids"); + break; + default: + VIR_WARN("got unknown migration status'%s'", + qemuMonitorMigrationStatusTypeToString(status)); + break; + } + + return; +} + + static int qemuProcessHandleMigrationStatus(qemuMonitorPtr mon G_GNUC_UNUSED, virDomainObjPtr vm, @@ -1675,6 +1726,8 @@ qemuProcessHandleMigrationStatus(qemuMonitorPtr mon G_GNUC_UNUSED, } } + qemuProcessHandleMigrationPinStatus(priv, status); + cleanup: virObjectUnlock(vm); virObjectEventStateQueue(driver->domainEventState, event); @@ -1915,6 +1968,7 @@ static qemuMonitorCallbacks monitorCallbacks = { .domainRdmaGidStatusChanged = qemuProcessHandleRdmaGidStatusChanged, .domainGuestCrashloaded = qemuProcessHandleGuestCrashloaded, .domainMigrationPid = qemuProcessHandleMigrationPid, + .domainMigrationMultiFdPids = qemuProcessHandleMigrationMultiFdPids, }; static void @@ -2971,6 +3025,71 @@ qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED, } +int +qemuProcessHandleMigrationMultiFdPids(qemuMonitorPtr mon ATTRIBUTE_UNUSED, + virDomainObjPtr vm, + int mpid, + void *opaque G_GNUC_UNUSED) +{ + qemuDomainObjPrivatePtr priv; + char *mpidOldStr = NULL; + char *mpidStr = NULL; + virDomainMigrationIDDefPtr migration = NULL; + virBitmapPtr pcpumap = NULL; + virObjectLock(vm); + + VIR_INFO("Migrating domain %p %s, migration-multifd pid %d", + vm, vm->def->name, mpid); + + priv = vm->privateData; + if (priv->job.asyncJob == QEMU_ASYNC_JOB_NONE) { + VIR_DEBUG("got MIGRATION_MULTIFD_PID event without a migration job"); + goto cleanup; + } + + if (VIR_ALLOC(migration) < 0) { + VIR_ERROR(_("alloc migrationIDDefPtr failure")); + goto cleanup; + } + migration->thread_id = mpid; + + if (qemuProcessSetupMigration(vm, migration) < 0) { + VIR_ERROR(_("fail to setup migration multiFd cgroup")); + goto cleanup; + } + + mpidOldStr = priv->migrationMultiFdPids; + if (!mpidOldStr) { + mpidStr = g_strdup_printf("%d", mpid); + } else { + mpidStr = g_strdup_printf("%s/%d", mpidOldStr, mpid); + } + + VIR_FREE(priv->migrationMultiFdPids); + priv->migrationMultiFdPids = mpidStr; + priv->migrationMultiFdCount++; + + pcpumap = qemuProcessGetPcpumap(priv); + + if (!pcpumap) + goto cleanup; + + qemuProcessSetMigthreadAffinity(priv, pcpumap, mpid); + + cleanup: + /* + * If the value of pcpumap is setted by priv->migrationThreadPinList, + * we need to free pcpumap. + */ + if (pcpumap != priv->pcpumap) + virBitmapFree(pcpumap); + virDomainMigrationIDDefFree(migration); + virObjectUnlock(vm); + + return 0; +} + + static char * qemuProcessBuildPRHelperPidfilePath(virDomainObjPtr vm) { diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index be062b9f95..561795edeb 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -249,3 +249,8 @@ int qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED, virDomainObjPtr vm, int mpid, void *opaque); + +int qemuProcessHandleMigrationMultiFdPids(qemuMonitorPtr mon ATTRIBUTE_UNUSED, + virDomainObjPtr vm, + int mpid, + void *opaque); -- 2.25.1