packages/libvirt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!199 libvirt增加迁移绑核功能

Browse Source 
From: @Mayunlong541 
Reviewed-by: @yezengruan 
Signed-off-by: @yezengruan
This commit is contained in:
openeuler-ci-bot 2022-12-17 06:11:48 +00:00 committed by Gitee
commit 3e5cb55b1a
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
7 changed files with 1010 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
libvirt.spec
View File

@ -101,7 +101,7 @@
Summary: Library providing a simple virtualization API
Name: libvirt
Version: 6.2.0
Release: 48
Release: 49
License: LGPLv2+
URL: https://libvirt.org/
@ -426,6 +426,12 @@ Patch0313: qemu_capabilities-Introduce-QEMU_CAPS_DIRTYRATE_MODE.patch
Patch0314: include-Introduce-virDomainDirtyRateCalcFlags.patch
Patch0315: qemu_driver-Add-mode-parameter-to-qemuDomainStartDir.patch
Patch0316: update-the-Chinese-translation-of-nwfilter.patch
Patch0317: migration-migration-pin-add-some-migration-multiFd-p.patch
Patch0318: migration-migration-pin-add-qemu-monitor-callback-fu.patch
Patch0319: migration-migration-pin-add-migrationpin-for-migrati.patch
Patch0320: migration-migration-pin-add-domainMigrationPid-for-q.patch
Patch0321: migration-multifd-pin-add-qemu-monitor-callback-func.patch
Patch0322: migration-multifd-pin-support-migration-multifd-thre.patch
Requires: libvirt-daemon = %{version}-%{release}
Requires: libvirt-daemon-config-network = %{version}-%{release}
@ -2160,6 +2166,9 @@ exit 0
%changelog
* Sat Dec 17 2022 zhengchuan <zhengchuan@huawei.com> - 6.2.0-49
- add function of set migration thread affinity during migration
* Sat Dec 10 2022 yezengruan <yezengruan@huawei.com> - 6.2.0-48
- update the Chinese translation of nwfilter

252
migration-migration-pin-add-domainMigrationPid-for-q.patch Normal file
View File

@ -0,0 +1,252 @@
From 18b3736c7a0dd1915401981750a5d1a3b3560031 Mon Sep 17 00:00:00 2001
From: zhengchuan <zhengchuan@huawei.com>
Date: Wed, 30 Nov 2022 15:01:12 +0800
Subject: [PATCH 4/6] migration/migration-pin: add domainMigrationPid for
qemuMonitorCallbacks
add domainMigrationPid for qemuMonitorCallbacks
Signed-off-by:zhengchuan<zhengchuan@huawei.com>
---
src/qemu/qemu_process.c | 189 ++++++++++++++++++++++++++++++++++++++++
src/qemu/qemu_process.h | 13 +++
2 files changed, 202 insertions(+)
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 9cbf34a046..ce1c597da1 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -1914,6 +1914,7 @@ static qemuMonitorCallbacks monitorCallbacks = {
.domainPRManagerStatusChanged = qemuProcessHandlePRManagerStatusChanged,
.domainRdmaGidStatusChanged = qemuProcessHandleRdmaGidStatusChanged,
.domainGuestCrashloaded = qemuProcessHandleGuestCrashloaded,
+ .domainMigrationPid = qemuProcessHandleMigrationPid,
};
static void
@@ -2782,6 +2783,194 @@ qemuProcessResctrlCreate(virQEMUDriverPtr driver,
}
+int
+qemuProcessSetupMigration(virDomainObjPtr vm,
+ virDomainMigrationIDDefPtr migration)
+{
+ return qemuProcessSetupPid(vm, migration->thread_id,
+ VIR_CGROUP_THREAD_MIGRATION_THREAD,
+ 0,
+ vm->def->cputune.emulatorpin,
+ vm->def->cputune.emulator_period,
+ vm->def->cputune.emulator_quota,
+ &migration->sched);
+}
+
+
+unsigned char *
+virParseCPUList(int *cpumaplen, const char *cpulist, int maxcpu)
+{
+ unsigned char *cpumap = NULL;
+ virBitmapPtr map = NULL;
+
+ if (cpulist[0] == 'r') {
+ map = virBitmapNew(maxcpu);
+ if (!map)
+ return NULL;
+ virBitmapSetAll(map);
+ } else {
+ if (virBitmapParse(cpulist, &map, 1024) < 0 ||
+ virBitmapIsAllClear(map)) {
+ goto cleanup;
+ }
+
+ int lastcpu = virBitmapLastSetBit(map);
+ if (lastcpu >= maxcpu)
+ goto cleanup;
+ }
+
+ if (virBitmapToData(map, &cpumap, cpumaplen) < 0)
+ VIR_ERROR(_("Bitmap to data failure"));
+
+ cleanup:
+ virBitmapFree(map);
+ return cpumap;
+}
+
+
+/*
+ * If priv->pcpumap is NULL, it means migrationpin command is not called,
+ * otherwise we set the affinity of migration thread by migrationpin.
+ */
+static virBitmapPtr
+qemuProcessGetPcpumap(qemuDomainObjPrivatePtr priv)
+{
+ int cpumaplen = 0;
+ int maxcpu = 0;
+ g_autofree unsigned char *cpumap = NULL;
+ virBitmapPtr pcpumap = NULL;
+
+ if(priv->pcpumap)
+ return priv->pcpumap;
+
+ if (!(priv->migrationThreadPinList) || STREQ(priv->migrationThreadPinList, "")) {
+ VIR_ERROR(_("didn't set the migratin thread pin"));
+ return NULL;
+ }
+
+ /* judge whether migration.pin is default value or not */
+ if (STREQ(priv->migrationThreadPinList, "none"))
+ return NULL;
+
+ maxcpu = virHostCPUGetCount();
+ if (maxcpu < 0) {
+ VIR_ERROR(_("get the cpu count of host failure"));
+ return NULL;
+ }
+
+ cpumap = virParseCPUList(&cpumaplen, priv->migrationThreadPinList, maxcpu);
+ if (!cpumap) {
+ VIR_ERROR(_("parse migration.pin params failure : migration.pin = %s"),
+ priv->migrationThreadPinList);
+ return NULL;
+ }
+
+ if (!(pcpumap = virBitmapNewData(cpumap, cpumaplen))) {
+ VIR_ERROR(_("Bitmap data failure"));
+ return pcpumap;
+ }
+
+ return pcpumap;
+}
+
+
+/*
+ * In order to set migration thread affinity when vm is migrating,
+ * we should create the cgroup for migration thread.
+ */
+static void
+qemuProcessSetMigthreadAffinity(qemuDomainObjPrivatePtr priv,
+ virBitmapPtr pcpumap,
+ int mpid)
+{
+ int migration_id = 0;
+ virCgroupPtr cgroup_migthread = NULL;
+
+ if (!pcpumap)
+ return;
+
+ if (virCgroupHasController(priv->cgroup,
+ VIR_CGROUP_CONTROLLER_CPUSET)) {
+ if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD,
+ migration_id, false, &cgroup_migthread) < 0)
+ goto cleanup;
+
+ if (qemuSetupCgroupCpusetCpus(cgroup_migthread, pcpumap) < 0) {
+ virReportError(VIR_ERR_OPERATION_INVALID,
+ _("failed to set cpuset.cpus in cgroup"
+ " for migration%d thread"), migration_id);
+ goto cleanup;
+ }
+ }
+
+ if (virProcessSetAffinity(mpid, pcpumap) < 0)
+ VIR_WARN("failed to set affinity in migration");
+
+ cleanup:
+ if (cgroup_migthread)
+ virCgroupFree(&cgroup_migthread);
+ return;
+}
+
+
+int
+qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
+ virDomainObjPtr vm,
+ int mpid,
+ void *opaque G_GNUC_UNUSED)
+{
+ qemuDomainObjPrivatePtr priv;
+ char *mpidStr = NULL;
+ virDomainMigrationIDDefPtr migration = NULL;
+ virBitmapPtr pcpumap = NULL;
+ virObjectLock(vm);
+
+ VIR_INFO("Migrating domain %p %s, migration pid %d",
+ vm, vm->def->name, mpid);
+
+ priv = vm->privateData;
+ if (priv->job.asyncJob == QEMU_ASYNC_JOB_NONE) {
+ VIR_DEBUG("got MIGRATION_PID event without a migration job");
+ goto cleanup;
+ }
+
+ if (VIR_ALLOC(migration) < 0) {
+ VIR_ERROR(_("alloc migrationIDDefPtr failure"));
+ goto cleanup;
+ }
+ migration->thread_id = mpid;
+
+ if (qemuProcessSetupMigration(vm, migration) < 0) {
+ VIR_ERROR(_("fail to setup migration cgroup"));
+ goto cleanup;
+ }
+
+ mpidStr = g_strdup_printf("%d", mpid);
+
+ VIR_FREE(priv->migrationPids);
+ priv->migrationPids = mpidStr;
+
+ pcpumap = qemuProcessGetPcpumap(priv);
+
+ if (!pcpumap)
+ goto cleanup;
+
+ qemuProcessSetMigthreadAffinity(priv, pcpumap, mpid);
+
+ cleanup:
+ /*
+ * If the value of pcpumap is setted by priv->migrationThreadPinList,
+ * we need to free pcpumap.
+ */
+ if (pcpumap != priv->pcpumap)
+ virBitmapFree(pcpumap);
+ virDomainMigrationIDDefFree(migration);
+ virObjectUnlock(vm);
+
+ return 0;
+}
+
+
static char *
qemuProcessBuildPRHelperPidfilePath(virDomainObjPtr vm)
{
diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h
index 15e67b9762..be062b9f95 100644
--- a/src/qemu/qemu_process.h
+++ b/src/qemu/qemu_process.h
@@ -25,6 +25,7 @@
#include "qemu_domain.h"
#include "virstoragefile.h"
#include "vireventthread.h"
+#include "domain_conf.h"
int qemuProcessPrepareMonitorChr(virDomainChrSourceDefPtr monConfig,
const char *domainDir);
@@ -236,3 +237,15 @@ qemuProcessQMPPtr qemuProcessQMPNew(const char *binary,
void qemuProcessQMPFree(qemuProcessQMPPtr proc);
int qemuProcessQMPStart(qemuProcessQMPPtr proc);
+
+int qemuProcessSetupMigration(virDomainObjPtr vm,
+ virDomainMigrationIDDefPtr migration);
+
+unsigned char * virParseCPUList(int *cpumaplen,
+ const char *cpulist,
+ int maxcpu);
+
+int qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
+ virDomainObjPtr vm,
+ int mpid,
+ void *opaque);
--
2.25.1

180
migration-migration-pin-add-migrationpin-for-migrati.patch Normal file
View File

@ -0,0 +1,180 @@
From 4483bab5e3d6b031c851e6b10323a37818f147e4 Mon Sep 17 00:00:00 2001
From: zhengchuan <zhengchuan@huawei.com>
Date: Wed, 30 Nov 2022 15:01:12 +0800
Subject: [PATCH 3/6] migration/migration-pin: add migrationpin for migration
parameters
Add a migrationpin to the migration parameters of live migration to bind cores
to the migration thread during VM migration.
Signed-off-by:zhengchuan<zhengchuan@huawei.com>
---
include/libvirt/libvirt-domain.h | 8 ++++++++
src/qemu/qemu_domain.c | 1 +
src/qemu/qemu_domain.h | 1 +
src/qemu/qemu_migration.c | 3 +++
src/qemu/qemu_migration.h | 1 +
src/qemu/qemu_migration_params.c | 20 ++++++++++++++++++++
src/qemu/qemu_migration_params.h | 4 ++++
7 files changed, 38 insertions(+)
diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h
index c7956c81d6..8fe723198d 100644
--- a/include/libvirt/libvirt-domain.h
+++ b/include/libvirt/libvirt-domain.h
@@ -1016,6 +1016,14 @@ typedef enum {
*/
# define VIR_MIGRATE_PARAM_COMPRESSION_MT_DTHREADS "compression.mt.dthreads"
+/**
+ * VIR_MIGRATE_PARAM_MIGRATIONPIN:
+ *
+ * virDomainMigrate* params field: the pin of migration threads for
+ * migration as VIR_TYPED_PARAM_STRING.
+ */
+# define VIR_MIGRATE_PARAM_MIGRATIONPIN "migration.pin"
+
/**
* VIR_MIGRATE_PARAM_COMPRESSION_XBZRLE_CACHE:
*
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index 71d034fa42..b7fb4eb9f9 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -2323,6 +2323,7 @@ qemuDomainObjPrivateFree(void *data)
VIR_FREE(priv->lockState);
VIR_FREE(priv->origname);
VIR_FREE(priv->migrationPids);
+ VIR_FREE(priv->migrationThreadPinList);
virBitmapFree(priv->pcpumap);
VIR_FREE(priv->migrationMultiFdPids);
diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
index 38a842ebf6..7603724ccd 100644
--- a/src/qemu/qemu_domain.h
+++ b/src/qemu/qemu_domain.h
@@ -333,6 +333,7 @@ struct _qemuDomainObjPrivate {
char *origname;
int nbdPort; /* Port used for migration with NBD */
char *migrationPids;
+ char *migrationThreadPinList;
unsigned short migrationPort;
int preMigrationState;
char *migrationMultiFdPids;
diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index faf4e223a8..b47e69c642 100644
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -2533,6 +2533,7 @@ qemuMigrationDstPrepareAny(virQEMUDriverPtr driver,
priv = vm->privateData;
priv->origname = g_strdup(origname);
+ VIR_FREE(priv->migrationPids);
if (taint_hook) {
/* Domain XML has been altered by a hook script. */
@@ -3601,6 +3602,8 @@ qemuMigrationSrcRun(virQEMUDriverPtr driver,
migrate_speed * 1024 * 1024) < 0)
goto error;
+ qemuMigrationMigrationParamsToVM(migParams, vm);
+
if (qemuMigrationParamsApply(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT,
migParams) < 0)
goto error;
diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h
index b6f88d3fd9..316a1a10f4 100644
--- a/src/qemu/qemu_migration.h
+++ b/src/qemu/qemu_migration.h
@@ -84,6 +84,7 @@
VIR_MIGRATE_PARAM_BANDWIDTH_POSTCOPY, VIR_TYPED_PARAM_ULLONG, \
VIR_MIGRATE_PARAM_PARALLEL_CONNECTIONS, VIR_TYPED_PARAM_INT, \
VIR_MIGRATE_PARAM_TLS_DESTINATION, VIR_TYPED_PARAM_STRING, \
+ VIR_MIGRATE_PARAM_MIGRATIONPIN, VIR_TYPED_PARAM_STRING, \
NULL
diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c
index df9d5d205a..4001fb2b34 100644
--- a/src/qemu/qemu_migration_params.c
+++ b/src/qemu/qemu_migration_params.c
@@ -108,6 +108,7 @@ VIR_ENUM_IMPL(qemuMigrationParam,
"xbzrle-cache-size",
"max-postcopy-bandwidth",
"multifd-channels",
+ "migrationpin",
);
typedef struct _qemuMigrationParamsAlwaysOnItem qemuMigrationParamsAlwaysOnItem;
@@ -201,6 +202,10 @@ static const qemuMigrationParamsTPMapItem qemuMigrationParamsTPMap[] = {
{.typedParam = VIR_MIGRATE_PARAM_TLS_DESTINATION,
.param = QEMU_MIGRATION_PARAM_TLS_HOSTNAME,
.party = QEMU_MIGRATION_SOURCE},
+
+ {.typedParam = VIR_MIGRATE_PARAM_MIGRATIONPIN,
+ .param = QEMU_MIGRATION_PARAM_MIGRATIONPIN,
+ .party = QEMU_MIGRATION_SOURCE},
};
static const qemuMigrationParamType qemuMigrationParamTypes[] = {
@@ -217,6 +222,7 @@ static const qemuMigrationParamType qemuMigrationParamTypes[] = {
[QEMU_MIGRATION_PARAM_XBZRLE_CACHE_SIZE] = QEMU_MIGRATION_PARAM_TYPE_ULL,
[QEMU_MIGRATION_PARAM_MAX_POSTCOPY_BANDWIDTH] = QEMU_MIGRATION_PARAM_TYPE_ULL,
[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS] = QEMU_MIGRATION_PARAM_TYPE_INT,
+ [QEMU_MIGRATION_PARAM_MIGRATIONPIN] = QEMU_MIGRATION_PARAM_TYPE_STRING,
};
G_STATIC_ASSERT(G_N_ELEMENTS(qemuMigrationParamTypes) == QEMU_MIGRATION_PARAM_LAST);
@@ -534,6 +540,16 @@ qemuMigrationParamsSetCompression(virTypedParameterPtr params,
}
+void
+qemuMigrationMigrationParamsToVM(const qemuMigrationParamsPtr migParams, const virDomainObjPtr vm)
+{
+ if (migParams && migParams->params[QEMU_MIGRATION_PARAM_MIGRATIONPIN].set) {
+ qemuDomainObjPrivatePtr priv = vm->privateData;
+ priv->migrationThreadPinList = g_strdup(migParams->params[QEMU_MIGRATION_PARAM_MIGRATIONPIN].value.s);
+ }
+}
+
+
qemuMigrationParamsPtr
qemuMigrationParamsFromFlags(virTypedParameterPtr params,
int nparams,
@@ -741,6 +757,10 @@ qemuMigrationParamsToJSON(qemuMigrationParamsPtr migParams)
if (!pv->set)
continue;
+ if (i == QEMU_MIGRATION_PARAM_MIGRATIONPIN) {
+ continue;
+ }
+
rc = 0;
switch (qemuMigrationParamTypes[i]) {
case QEMU_MIGRATION_PARAM_TYPE_INT:
diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h
index 9aea24725f..f3deffb4cc 100644
--- a/src/qemu/qemu_migration_params.h
+++ b/src/qemu/qemu_migration_params.h
@@ -58,6 +58,7 @@ typedef enum {
QEMU_MIGRATION_PARAM_XBZRLE_CACHE_SIZE,
QEMU_MIGRATION_PARAM_MAX_POSTCOPY_BANDWIDTH,
QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS,
+ QEMU_MIGRATION_PARAM_MIGRATIONPIN,
QEMU_MIGRATION_PARAM_LAST
} qemuMigrationParam;
@@ -74,6 +75,9 @@ typedef enum {
virBitmapPtr
qemuMigrationParamsGetAlwaysOnCaps(qemuMigrationParty party);
+void
+qemuMigrationMigrationParamsToVM(const qemuMigrationParamsPtr migParams, const virDomainObjPtr vm);
+
qemuMigrationParamsPtr
qemuMigrationParamsFromFlags(virTypedParameterPtr params,
int nparams,
--
2.25.1

114
migration-migration-pin-add-qemu-monitor-callback-fu.patch Normal file
View File

@ -0,0 +1,114 @@
From b0035c45ecfed66f0406a6064b33f296fd65afd9 Mon Sep 17 00:00:00 2001
From: zhengchuan <zhengchuan@huawei.com>
Date: Wed, 30 Nov 2022 11:30:12 +0800
Subject: [PATCH 2/6] migration/migration-pin: add qemu monitor callback
functions
add qemu monitor callback functions
Signed-off-by:zhengchuan<zhengchuan@huawei.com>
---
src/qemu/qemu_monitor.c | 12 ++++++++++++
src/qemu/qemu_monitor.h | 8 ++++++++
src/qemu/qemu_monitor_json.c | 15 +++++++++++++++
3 files changed, 35 insertions(+)
diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c
index 04592fdcd5..fcda2c0771 100644
--- a/src/qemu/qemu_monitor.c
+++ b/src/qemu/qemu_monitor.c
@@ -1547,6 +1547,18 @@ qemuMonitorEmitPRManagerStatusChanged(qemuMonitorPtr mon,
}
+int
+qemuMonitorEmitMigrationPid(qemuMonitorPtr mon,
+ int mpid)
+{
+ int ret = -1;
+ VIR_DEBUG("mon=%p, pass=%d", mon, mpid);
+ QEMU_MONITOR_CALLBACK(mon, ret, domainMigrationPid, mon->vm, mpid);
+
+ return ret;
+}
+
+
int
qemuMonitorEmitRdmaGidStatusChanged(qemuMonitorPtr mon,
const char *netdev,
diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h
index 4f9b70d820..4b34ea3b4a 100644
--- a/src/qemu/qemu_monitor.h
+++ b/src/qemu/qemu_monitor.h
@@ -340,6 +340,11 @@ typedef int (*qemuMonitorDomainGuestCrashloadedCallback)(qemuMonitorPtr mon,
virDomainObjPtr vm,
void *opaque);
+typedef int (*qemuMonitorDomainMigrationPidCallback)(qemuMonitorPtr mon,
+ virDomainObjPtr vm,
+ int mcpid,
+ void *opaque);
+
typedef struct _qemuMonitorCallbacks qemuMonitorCallbacks;
typedef qemuMonitorCallbacks *qemuMonitorCallbacksPtr;
struct _qemuMonitorCallbacks {
@@ -376,6 +381,7 @@ struct _qemuMonitorCallbacks {
qemuMonitorDomainPRManagerStatusChangedCallback domainPRManagerStatusChanged;
qemuMonitorDomainRdmaGidStatusChangedCallback domainRdmaGidStatusChanged;
qemuMonitorDomainGuestCrashloadedCallback domainGuestCrashloaded;
+ qemuMonitorDomainMigrationPidCallback domainMigrationPid;
};
qemuMonitorPtr qemuMonitorOpen(virDomainObjPtr vm,
@@ -477,6 +483,8 @@ int qemuMonitorEmitMigrationStatus(qemuMonitorPtr mon,
int qemuMonitorEmitMigrationPass(qemuMonitorPtr mon,
int pass);
+int qemuMonitorEmitMigrationPid(qemuMonitorPtr mon, int mpid);
+
int qemuMonitorEmitAcpiOstInfo(qemuMonitorPtr mon,
const char *alias,
const char *slotType,
diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c
index a1617cdfed..7d2fbec5f4 100644
--- a/src/qemu/qemu_monitor_json.c
+++ b/src/qemu/qemu_monitor_json.c
@@ -112,6 +112,7 @@ static void qemuMonitorJSONHandleBlockThreshold(qemuMonitorPtr mon, virJSONValue
static void qemuMonitorJSONHandleDumpCompleted(qemuMonitorPtr mon, virJSONValuePtr data);
static void qemuMonitorJSONHandlePRManagerStatusChanged(qemuMonitorPtr mon, virJSONValuePtr data);
static void qemuMonitorJSONHandleRdmaGidStatusChanged(qemuMonitorPtr mon, virJSONValuePtr data);
+static void qemuMonitorJSONHandleMigrationPid(qemuMonitorPtr mon, virJSONValuePtr data);
typedef struct {
const char *type;
@@ -134,6 +135,7 @@ static qemuEventHandler eventHandlers[] = {
{ "JOB_STATUS_CHANGE", qemuMonitorJSONHandleJobStatusChange, },
{ "MIGRATION", qemuMonitorJSONHandleMigrationStatus, },
{ "MIGRATION_PASS", qemuMonitorJSONHandleMigrationPass, },
+ { "MIGRATION_PID", qemuMonitorJSONHandleMigrationPid, },
{ "NIC_RX_FILTER_CHANGED", qemuMonitorJSONHandleNicRxFilterChanged, },
{ "POWERDOWN", qemuMonitorJSONHandlePowerdown, },
{ "PR_MANAGER_STATUS_CHANGED", qemuMonitorJSONHandlePRManagerStatusChanged, },
@@ -158,6 +160,19 @@ static qemuEventHandler eventHandlers[] = {
/* We use bsearch, so keep this list sorted. */
};
+static void qemuMonitorJSONHandleMigrationPid(qemuMonitorPtr mon,
+ virJSONValuePtr data)
+{
+ int mpid;
+
+ if (virJSONValueObjectGetNumberInt(data, "pid", &mpid) < 0) {
+ VIR_WARN("missing migration pid in migration-pid event");
+ return;
+ }
+
+ qemuMonitorEmitMigrationPid(mon, mpid);
+}
+
static int
qemuMonitorEventCompare(const void *key, const void *elt)
{
--
2.25.1

144
migration-migration-pin-add-some-migration-multiFd-p.patch Normal file
View File

@ -0,0 +1,144 @@
From 36295c0cc955fee9afc7b0fb70822c8e1e030b87 Mon Sep 17 00:00:00 2001
From: zhengchuan <zhengchuan@huawei.com>
Date: Wed, 14 Dec 2022 11:07:58 +0800
Subject: [PATCH 1/6] migration/migration-pin:add some migration/multiFd params
add migration/multiFd params.
Signed-off-by:zhengchuan<zhengchuan@huawei.com>
---
src/conf/domain_conf.c | 9 +++++++++
src/conf/domain_conf.h | 12 ++++++++++++
src/libvirt_private.syms | 1 +
src/qemu/qemu_domain.c | 6 ++++++
src/qemu/qemu_domain.h | 4 ++++
src/util/vircgroup.c | 3 +++
src/util/vircgroup.h | 1 +
7 files changed, 36 insertions(+)
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 8b1ca76d39..1689d92c51 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -32617,3 +32617,12 @@ virHostdevIsVFIODevice(const virDomainHostdevDef *hostdev)
hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI &&
hostdev->source.subsys.u.pci.backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO;
}
+
+void
+virDomainMigrationIDDefFree(virDomainMigrationIDDefPtr def)
+{
+ if (!def)
+ return;
+ virBitmapFree(def->cpumask);
+ VIR_FREE(def);
+}
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index c0a323d465..98fa323679 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -27,6 +27,7 @@
#include <libxml/xpath.h>
#include "internal.h"
+#include "viralloc.h"
#include "virconftypes.h"
#include "capabilities.h"
#include "virstorageencryption.h"
@@ -3775,3 +3776,14 @@ virHostdevIsMdevDevice(const virDomainHostdevDef *hostdev)
bool
virHostdevIsVFIODevice(const virDomainHostdevDef *hostdev)
ATTRIBUTE_NONNULL(1);
+
+typedef struct _virDomainMigrationIDDef virDomainMigrationIDDef;
+typedef virDomainMigrationIDDef *virDomainMigrationIDDefPtr;
+struct _virDomainMigrationIDDef {
+ bool autofill;
+ int thread_id;
+ virBitmapPtr cpumask;
+ virDomainThreadSchedParam sched;
+};
+
+void virDomainMigrationIDDefFree(virDomainMigrationIDDefPtr def);
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index 14ad7ecde3..f30eb7ffb5 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -492,6 +492,7 @@ virDomainMemoryModelTypeToString;
virDomainMemoryRemove;
virDomainMemorySourceTypeFromString;
virDomainMemorySourceTypeToString;
+virDomainMigrationIDDefFree;
virDomainNetAllocateActualDevice;
virDomainNetAppendIPAddress;
virDomainNetARPInterfaces;
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index b2ac1d37cc..71d034fa42 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -2322,6 +2322,12 @@ qemuDomainObjPrivateFree(void *data)
qemuDomainObjFreeJob(priv);
VIR_FREE(priv->lockState);
VIR_FREE(priv->origname);
+ VIR_FREE(priv->migrationPids);
+ virBitmapFree(priv->pcpumap);
+
+ VIR_FREE(priv->migrationMultiFdPids);
+ priv->migrationMultiFdPids = NULL;
+ priv->migrationMultiFdCount = 0;
virChrdevFree(priv->devs);
diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
index 164fb36f0c..38a842ebf6 100644
--- a/src/qemu/qemu_domain.h
+++ b/src/qemu/qemu_domain.h
@@ -332,8 +332,11 @@ struct _qemuDomainObjPrivate {
unsigned long migMaxBandwidth;
char *origname;
int nbdPort; /* Port used for migration with NBD */
+ char *migrationPids;
unsigned short migrationPort;
int preMigrationState;
+ char *migrationMultiFdPids;
+ unsigned int migrationMultiFdCount;
virChrdevsPtr devs;
@@ -354,6 +357,7 @@ struct _qemuDomainObjPrivate {
/* Bitmaps below hold data from the auto NUMA feature */
virBitmapPtr autoNodeset;
virBitmapPtr autoCpuset;
+ virBitmapPtr pcpumap;
bool signalIOError; /* true if the domain condition should be signalled on
I/O error */
diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c
index f52a8cc843..b075dd1ecb 100644
--- a/src/util/vircgroup.c
+++ b/src/util/vircgroup.c
@@ -969,6 +969,9 @@ virCgroupNewThread(virCgroupPtr domain,
case VIR_CGROUP_THREAD_IOTHREAD:
name = g_strdup_printf("iothread%d", id);
break;
+ case VIR_CGROUP_THREAD_MIGRATION_THREAD:
+ name = g_strdup_printf("migthread%d", id);
+ break;
case VIR_CGROUP_THREAD_LAST:
virReportError(VIR_ERR_INTERNAL_ERROR,
_("unexpected name value %d"), nameval);
diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h
index 1dcd0688f1..5639f36c1f 100644
--- a/src/util/vircgroup.h
+++ b/src/util/vircgroup.h
@@ -54,6 +54,7 @@ typedef enum {
VIR_CGROUP_THREAD_VCPU = 0,
VIR_CGROUP_THREAD_EMULATOR,
VIR_CGROUP_THREAD_IOTHREAD,
+ VIR_CGROUP_THREAD_MIGRATION_THREAD,
VIR_CGROUP_THREAD_LAST
} virCgroupThreadName;
--
2.25.1

115
migration-multifd-pin-add-qemu-monitor-callback-func.patch Normal file
View File

@ -0,0 +1,115 @@
From d0c3db4eb01ac0c31003c333173aa2b8dceb8032 Mon Sep 17 00:00:00 2001
From: zhengchuan <zhengchuan@huawei.com>
Date: Wed, 30 Nov 2022 15:59:27 +0800
Subject: [PATCH 5/6] migration/multifd-pin: add qemu monitor callback
functions
add qemu monitor callback functions
Signed-off-by:zhengchuan<zhengchuan@huawei.com>
---
src/qemu/qemu_monitor.c | 12 ++++++++++++
src/qemu/qemu_monitor.h | 9 +++++++++
src/qemu/qemu_monitor_json.c | 15 +++++++++++++++
3 files changed, 36 insertions(+)
diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c
index fcda2c0771..ffd1d348e5 100644
--- a/src/qemu/qemu_monitor.c
+++ b/src/qemu/qemu_monitor.c
@@ -1559,6 +1559,18 @@ qemuMonitorEmitMigrationPid(qemuMonitorPtr mon,
}
+int
+qemuMonitorEmitMigrationMultiFdPids(qemuMonitorPtr mon,
+ int mpid)
+{
+ int ret = -1;
+ VIR_DEBUG("mon=%p, pass=%d", mon, mpid);
+ QEMU_MONITOR_CALLBACK(mon, ret, domainMigrationMultiFdPids, mon->vm, mpid);
+
+ return ret;
+}
+
+
int
qemuMonitorEmitRdmaGidStatusChanged(qemuMonitorPtr mon,
const char *netdev,
diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h
index 4b34ea3b4a..76d0bbb753 100644
--- a/src/qemu/qemu_monitor.h
+++ b/src/qemu/qemu_monitor.h
@@ -345,6 +345,11 @@ typedef int (*qemuMonitorDomainMigrationPidCallback)(qemuMonitorPtr mon,
int mcpid,
void *opaque);
+typedef int (*qemuMonitorDomainMigrationMultiFdPidsCallback)(qemuMonitorPtr mon,
+ virDomainObjPtr vm,
+ int mcpid,
+ void *opaque);
+
typedef struct _qemuMonitorCallbacks qemuMonitorCallbacks;
typedef qemuMonitorCallbacks *qemuMonitorCallbacksPtr;
struct _qemuMonitorCallbacks {
@@ -382,6 +387,7 @@ struct _qemuMonitorCallbacks {
qemuMonitorDomainRdmaGidStatusChangedCallback domainRdmaGidStatusChanged;
qemuMonitorDomainGuestCrashloadedCallback domainGuestCrashloaded;
qemuMonitorDomainMigrationPidCallback domainMigrationPid;
+ qemuMonitorDomainMigrationMultiFdPidsCallback domainMigrationMultiFdPids;
};
qemuMonitorPtr qemuMonitorOpen(virDomainObjPtr vm,
@@ -485,6 +491,9 @@ int qemuMonitorEmitMigrationPass(qemuMonitorPtr mon,
int qemuMonitorEmitMigrationPid(qemuMonitorPtr mon, int mpid);
+int qemuMonitorEmitMigrationMultiFdPids(qemuMonitorPtr mon,
+ int mpid);
+
int qemuMonitorEmitAcpiOstInfo(qemuMonitorPtr mon,
const char *alias,
const char *slotType,
diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c
index 7d2fbec5f4..a02d0e2780 100644
--- a/src/qemu/qemu_monitor_json.c
+++ b/src/qemu/qemu_monitor_json.c
@@ -113,6 +113,7 @@ static void qemuMonitorJSONHandleDumpCompleted(qemuMonitorPtr mon, virJSONValueP
static void qemuMonitorJSONHandlePRManagerStatusChanged(qemuMonitorPtr mon, virJSONValuePtr data);
static void qemuMonitorJSONHandleRdmaGidStatusChanged(qemuMonitorPtr mon, virJSONValuePtr data);
static void qemuMonitorJSONHandleMigrationPid(qemuMonitorPtr mon, virJSONValuePtr data);
+static void qemuMonitorJSONHandleMigrationMultiFdPids(qemuMonitorPtr mon, virJSONValuePtr data);
typedef struct {
const char *type;
@@ -134,6 +135,7 @@ static qemuEventHandler eventHandlers[] = {
{ "GUEST_PANICKED", qemuMonitorJSONHandleGuestPanic, },
{ "JOB_STATUS_CHANGE", qemuMonitorJSONHandleJobStatusChange, },
{ "MIGRATION", qemuMonitorJSONHandleMigrationStatus, },
+ { "MIGRATION_MULTIFD_PID", qemuMonitorJSONHandleMigrationMultiFdPids, },
{ "MIGRATION_PASS", qemuMonitorJSONHandleMigrationPass, },
{ "MIGRATION_PID", qemuMonitorJSONHandleMigrationPid, },
{ "NIC_RX_FILTER_CHANGED", qemuMonitorJSONHandleNicRxFilterChanged, },
@@ -173,6 +175,19 @@ static void qemuMonitorJSONHandleMigrationPid(qemuMonitorPtr mon,
qemuMonitorEmitMigrationPid(mon, mpid);
}
+static void qemuMonitorJSONHandleMigrationMultiFdPids(qemuMonitorPtr mon,
+ virJSONValuePtr data)
+{
+ int mpid;
+
+ if (virJSONValueObjectGetNumberInt(data, "pid", &mpid) < 0) {
+ VIR_WARN("missing multifd pid in migration-multifd-pid event");
+ return;
+ }
+
+ qemuMonitorEmitMigrationMultiFdPids(mon, mpid);
+}
+
static int
qemuMonitorEventCompare(const void *key, const void *elt)
{
--
2.25.1

195
migration-multifd-pin-support-migration-multifd-thre.patch Normal file
View File

@ -0,0 +1,195 @@
From 8c07702d9280ef5235bb2bdb1ee472452cffca02 Mon Sep 17 00:00:00 2001
From: zhengchuan <zhengchuan@huawei.com>
Date: Wed, 30 Nov 2022 16:47:30 +0800
Subject: [PATCH 6/6] migration/multifd-pin: support migration multifd thread
pin
support migration multifd thread pin by configuration.
Signed-off-by:zhengchuan<zhengchuan@huawei.com>
---
src/qemu/qemu_migration.c | 2 +
src/qemu/qemu_process.c | 119 ++++++++++++++++++++++++++++++++++++++
src/qemu/qemu_process.h | 5 ++
3 files changed, 126 insertions(+)
diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index b47e69c642..c2a694eff1 100644
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -2534,6 +2534,8 @@ qemuMigrationDstPrepareAny(virQEMUDriverPtr driver,
priv = vm->privateData;
priv->origname = g_strdup(origname);
VIR_FREE(priv->migrationPids);
+ VIR_FREE(priv->migrationMultiFdPids);
+ priv->migrationMultiFdCount = 0;
if (taint_hook) {
/* Domain XML has been altered by a hook script. */
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index ce1c597da1..0f3e651630 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -1629,6 +1629,57 @@ qemuProcessHandleSpiceMigrated(qemuMonitorPtr mon G_GNUC_UNUSED,
}
+static void
+qemuProcessHandleMigrationPinStatus(qemuDomainObjPrivatePtr priv, int status)
+{
+ if (priv->job.asyncJob != QEMU_ASYNC_JOB_MIGRATION_OUT)
+ return;
+
+ switch (status) {
+ case QEMU_MONITOR_MIGRATION_STATUS_INACTIVE:
+ case QEMU_MONITOR_MIGRATION_STATUS_SETUP:
+ case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE:
+ case QEMU_MONITOR_MIGRATION_STATUS_PRE_SWITCHOVER:
+ case QEMU_MONITOR_MIGRATION_STATUS_DEVICE:
+ case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY:
+ case QEMU_MONITOR_MIGRATION_STATUS_CANCELLING:
+ case QEMU_MONITOR_MIGRATION_STATUS_COMPLETED:
+ case QEMU_MONITOR_MIGRATION_STATUS_WAIT_UNPLUG:
+ break;
+ case QEMU_MONITOR_MIGRATION_STATUS_ERROR:
+ /*
+ * migration thread is still running,
+ * so we can't delete migration Cgroup.
+ */
+ VIR_FREE(priv->migrationPids);
+ VIR_FREE(priv->migrationMultiFdPids);
+ VIR_FREE(priv->migrationThreadPinList);
+ priv->migrationMultiFdCount = 0;
+ virBitmapFree(priv->pcpumap);
+ priv->pcpumap = NULL;
+ break;
+ case QEMU_MONITOR_MIGRATION_STATUS_CANCELLED:
+ VIR_FREE(priv->migrationPids);
+ VIR_FREE(priv->migrationMultiFdPids);
+ VIR_FREE(priv->migrationThreadPinList);
+ priv->migrationMultiFdCount = 0;
+ virBitmapFree(priv->pcpumap);
+ priv->pcpumap = NULL;
+ if (virCgroupDelThread(priv->cgroup,
+ VIR_CGROUP_THREAD_MIGRATION_THREAD, 0) < 0)
+ VIR_WARN("Failed to delete migration thread Cgroup!");
+ VIR_INFO("success to free pcpumap and migrationPids");
+ break;
+ default:
+ VIR_WARN("got unknown migration status'%s'",
+ qemuMonitorMigrationStatusTypeToString(status));
+ break;
+ }
+
+ return;
+}
+
+
static int
qemuProcessHandleMigrationStatus(qemuMonitorPtr mon G_GNUC_UNUSED,
virDomainObjPtr vm,
@@ -1675,6 +1726,8 @@ qemuProcessHandleMigrationStatus(qemuMonitorPtr mon G_GNUC_UNUSED,
}
}
+ qemuProcessHandleMigrationPinStatus(priv, status);
+
cleanup:
virObjectUnlock(vm);
virObjectEventStateQueue(driver->domainEventState, event);
@@ -1915,6 +1968,7 @@ static qemuMonitorCallbacks monitorCallbacks = {
.domainRdmaGidStatusChanged = qemuProcessHandleRdmaGidStatusChanged,
.domainGuestCrashloaded = qemuProcessHandleGuestCrashloaded,
.domainMigrationPid = qemuProcessHandleMigrationPid,
+ .domainMigrationMultiFdPids = qemuProcessHandleMigrationMultiFdPids,
};
static void
@@ -2971,6 +3025,71 @@ qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
}
+int
+qemuProcessHandleMigrationMultiFdPids(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
+ virDomainObjPtr vm,
+ int mpid,
+ void *opaque G_GNUC_UNUSED)
+{
+ qemuDomainObjPrivatePtr priv;
+ char *mpidOldStr = NULL;
+ char *mpidStr = NULL;
+ virDomainMigrationIDDefPtr migration = NULL;
+ virBitmapPtr pcpumap = NULL;
+ virObjectLock(vm);
+
+ VIR_INFO("Migrating domain %p %s, migration-multifd pid %d",
+ vm, vm->def->name, mpid);
+
+ priv = vm->privateData;
+ if (priv->job.asyncJob == QEMU_ASYNC_JOB_NONE) {
+ VIR_DEBUG("got MIGRATION_MULTIFD_PID event without a migration job");
+ goto cleanup;
+ }
+
+ if (VIR_ALLOC(migration) < 0) {
+ VIR_ERROR(_("alloc migrationIDDefPtr failure"));
+ goto cleanup;
+ }
+ migration->thread_id = mpid;
+
+ if (qemuProcessSetupMigration(vm, migration) < 0) {
+ VIR_ERROR(_("fail to setup migration multiFd cgroup"));
+ goto cleanup;
+ }
+
+ mpidOldStr = priv->migrationMultiFdPids;
+ if (!mpidOldStr) {
+ mpidStr = g_strdup_printf("%d", mpid);
+ } else {
+ mpidStr = g_strdup_printf("%s/%d", mpidOldStr, mpid);
+ }
+
+ VIR_FREE(priv->migrationMultiFdPids);
+ priv->migrationMultiFdPids = mpidStr;
+ priv->migrationMultiFdCount++;
+
+ pcpumap = qemuProcessGetPcpumap(priv);
+
+ if (!pcpumap)
+ goto cleanup;
+
+ qemuProcessSetMigthreadAffinity(priv, pcpumap, mpid);
+
+ cleanup:
+ /*
+ * If the value of pcpumap is setted by priv->migrationThreadPinList,
+ * we need to free pcpumap.
+ */
+ if (pcpumap != priv->pcpumap)
+ virBitmapFree(pcpumap);
+ virDomainMigrationIDDefFree(migration);
+ virObjectUnlock(vm);
+
+ return 0;
+}
+
+
static char *
qemuProcessBuildPRHelperPidfilePath(virDomainObjPtr vm)
{
diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h
index be062b9f95..561795edeb 100644
--- a/src/qemu/qemu_process.h
+++ b/src/qemu/qemu_process.h
@@ -249,3 +249,8 @@ int qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
virDomainObjPtr vm,
int mpid,
void *opaque);
+
+int qemuProcessHandleMigrationMultiFdPids(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
+ virDomainObjPtr vm,
+ int mpid,
+ void *opaque);
--
2.25.1