packages/qemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

migration: backport migration patches from upstream

Browse Source 
This series patches backport migration patches from upstream.
This commit is contained in:
zhengchuan 2020-04-24 15:37:03 +08:00 committed by Ying Fang
parent 1c823f0431
commit 36efc4445a
11 changed files with 1002 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
migration-Make-global-sem_sync-semaphore-by-channel.patch Normal file
View File

@ -0,0 +1,100 @@
From 8c3794d709eefdae777477bef7ff3511d55bf418 Mon Sep 17 00:00:00 2001
From: Juan Quintela <quintela@redhat.com>
Date: Wed, 14 Aug 2019 04:02:14 +0200
Subject: [PATCH 05/10] migration: Make global sem_sync semaphore by channel
This makes easy to debug things because when you want for all threads
to arrive at that semaphore, you know which one your are waiting for.
Change-Id: I533af8cdc68f619b68eff8e4e573c4de371a3954
Signed-off-by: Juan Quintela <quintela@redhat.com>
Message-Id: <20190814020218.1868-3-quintela@redhat.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
migration/ram.c | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/migration/ram.c b/migration/ram.c
index c75716bb..51811c2d 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -661,6 +661,8 @@ typedef struct {
uint64_t num_packets;
/* pages sent through this channel */
uint64_t num_pages;
+ /* syncs main thread and channels */
+ QemuSemaphore sem_sync;
} MultiFDSendParams;
typedef struct {
@@ -896,8 +898,6 @@ struct {
MultiFDSendParams *params;
/* array of pages to sent */
MultiFDPages_t *pages;
- /* syncs main thread and channels */
- QemuSemaphore sem_sync;
/* global number of generated multifd packets */
uint64_t packet_num;
/* send channels ready */
@@ -1037,6 +1037,7 @@ void multifd_save_cleanup(void)
p->c = NULL;
qemu_mutex_destroy(&p->mutex);
qemu_sem_destroy(&p->sem);
+ qemu_sem_destroy(&p->sem_sync);
g_free(p->name);
p->name = NULL;
multifd_pages_clear(p->pages);
@@ -1046,7 +1047,6 @@ void multifd_save_cleanup(void)
p->packet = NULL;
}
qemu_sem_destroy(&multifd_send_state->channels_ready);
- qemu_sem_destroy(&multifd_send_state->sem_sync);
g_free(multifd_send_state->params);
multifd_send_state->params = NULL;
multifd_pages_clear(multifd_send_state->pages);
@@ -1096,7 +1096,7 @@ static void multifd_send_sync_main(RAMState *rs)
MultiFDSendParams *p = &multifd_send_state->params[i];
trace_multifd_send_sync_main_wait(p->id);
- qemu_sem_wait(&multifd_send_state->sem_sync);
+ qemu_sem_wait(&p->sem_sync);
}
trace_multifd_send_sync_main(multifd_send_state->packet_num);
}
@@ -1156,7 +1156,7 @@ static void *multifd_send_thread(void *opaque)
qemu_mutex_unlock(&p->mutex);
if (flags & MULTIFD_FLAG_SYNC) {
- qemu_sem_post(&multifd_send_state->sem_sync);
+ qemu_sem_post(&p->sem_sync);
}
qemu_sem_post(&multifd_send_state->channels_ready);
} else if (p->quit) {
@@ -1179,7 +1179,7 @@ out:
*/
if (ret != 0) {
if (flags & MULTIFD_FLAG_SYNC) {
- qemu_sem_post(&multifd_send_state->sem_sync);
+ qemu_sem_post(&p->sem_sync);
}
qemu_sem_post(&multifd_send_state->channels_ready);
}
@@ -1225,7 +1225,6 @@ int multifd_save_setup(void)
multifd_send_state = g_malloc0(sizeof(*multifd_send_state));
multifd_send_state->params = g_new0(MultiFDSendParams, thread_count);
multifd_send_state->pages = multifd_pages_init(page_count);
- qemu_sem_init(&multifd_send_state->sem_sync, 0);
qemu_sem_init(&multifd_send_state->channels_ready, 0);
for (i = 0; i < thread_count; i++) {
@@ -1233,6 +1232,7 @@ int multifd_save_setup(void)
qemu_mutex_init(&p->mutex);
qemu_sem_init(&p->sem, 0);
+ qemu_sem_init(&p->sem_sync, 0);
p->quit = false;
p->pending_job = 0;
p->id = i;
--
2.19.1

57
migration-Maybe-VM-is-paused-when-migration-is-cance.patch Normal file
View File

@ -0,0 +1,57 @@
From 5e99e1329fa52dce8ab784a960e64a3e19b429aa Mon Sep 17 00:00:00 2001
From: Zhimin Feng <fengzhimin1@huawei.com>
Date: Tue, 14 Jan 2020 17:43:09 +0800
Subject: [PATCH 07/10] migration: Maybe VM is paused when migration is
cancelled
If the migration is cancelled when it is in the completion phase,
the migration state is set to MIGRATION_STATUS_CANCELLING.
The VM maybe wait for the 'pause_sem' semaphore in migration_maybe_pause
function, so that VM always is paused.
Change-Id: Ib2f2f42ee1edbb14da269ee19ba1fe16dd363822
Reported-by: Euler Robot <euler.robot@huawei.com>
Signed-off-by: Zhimin Feng <fengzhimin1@huawei.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
---
migration/migration.c | 24 ++++++++++++++++--------
1 file changed, 16 insertions(+), 8 deletions(-)
diff --git a/migration/migration.c b/migration/migration.c
index bea9b1d7..114c33a1 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -2731,14 +2731,22 @@ static int migration_maybe_pause(MigrationState *s,
/* This block intentionally left blank */
}
- qemu_mutex_unlock_iothread();
- migrate_set_state(&s->state, *current_active_state,
- MIGRATION_STATUS_PRE_SWITCHOVER);
- qemu_sem_wait(&s->pause_sem);
- migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
- new_state);
- *current_active_state = new_state;
- qemu_mutex_lock_iothread();
+ /*
+ * If the migration is cancelled when it is in the completion phase,
+ * the migration state is set to MIGRATION_STATUS_CANCELLING.
+ * So we don't need to wait a semaphore, otherwise we would always
+ * wait for the 'pause_sem' semaphore.
+ */
+ if (s->state != MIGRATION_STATUS_CANCELLING) {
+ qemu_mutex_unlock_iothread();
+ migrate_set_state(&s->state, *current_active_state,
+ MIGRATION_STATUS_PRE_SWITCHOVER);
+ qemu_sem_wait(&s->pause_sem);
+ migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
+ new_state);
+ *current_active_state = new_state;
+ qemu_mutex_lock_iothread();
+ }
return s->state == new_state ? 0 : -EINVAL;
}
--
2.19.1

50
migration-add-qemu_file_update_transfer-interface.patch Normal file
View File

@ -0,0 +1,50 @@
From 7572495245a437da717e6829a9ce852cc3f229c9 Mon Sep 17 00:00:00 2001
From: Zheng Chuan <zhengchuan@huawei.com>
Date: Mon, 20 Apr 2020 15:13:47 +0800
Subject: [PATCH 02/10] migration: add qemu_file_update_transfer interface
Add qemu_file_update_transfer for just update bytes_xfer for speed
limitation. This will be used for further migration feature such as
multifd migration.
Change-Id: I969aa15305c961254b6fb9805b0ed2d65826cc5d
Signed-off-by: Ivan Ren <ivanren@tencent.com>
Reviewed-by: Wei Yang <richardw.yang@linux.intel.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Message-Id: <1564464816-21804-2-git-send-email-ivanren@tencent.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
migration/qemu-file.c | 5 +++++
migration/qemu-file.h | 1 +
2 files changed, 6 insertions(+)
diff --git a/migration/qemu-file.c b/migration/qemu-file.c
index 04315855..18f48052 100644
--- a/migration/qemu-file.c
+++ b/migration/qemu-file.c
@@ -615,6 +615,11 @@ void qemu_file_reset_rate_limit(QEMUFile *f)
f->bytes_xfer = 0;
}
+void qemu_file_update_transfer(QEMUFile *f, int64_t len)
+{
+ f->bytes_xfer += len;
+}
+
void qemu_put_be16(QEMUFile *f, unsigned int v)
{
qemu_put_byte(f, v >> 8);
diff --git a/migration/qemu-file.h b/migration/qemu-file.h
index 13baf896..5de9fa2e 100644
--- a/migration/qemu-file.h
+++ b/migration/qemu-file.h
@@ -147,6 +147,7 @@ int qemu_peek_byte(QEMUFile *f, int offset);
void qemu_file_skip(QEMUFile *f, int size);
void qemu_update_position(QEMUFile *f, size_t size);
void qemu_file_reset_rate_limit(QEMUFile *f);
+void qemu_file_update_transfer(QEMUFile *f, int64_t len);
void qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate);
int64_t qemu_file_get_rate_limit(QEMUFile *f);
void qemu_file_set_error(QEMUFile *f, int ret);
--
2.19.1

127
migration-add-speed-limit-for-multifd-migration.patch Normal file
View File

@ -0,0 +1,127 @@
From bc5780480db9e38699df0b4697e60a9f36258dc4 Mon Sep 17 00:00:00 2001
From: Ivan Ren <ivanren@tencent.com>
Date: Tue, 30 Jul 2019 13:33:35 +0800
Subject: [PATCH 03/10] migration: add speed limit for multifd migration
Limit the speed of multifd migration through common speed limitation
qemu file.
Change-Id: Id2abfc7ea85679bd53130a43043cc70179a52e87
Signed-off-by: Ivan Ren <ivanren@tencent.com>
Message-Id: <1564464816-21804-3-git-send-email-ivanren@tencent.com>
Reviewed-by: Wei Yang <richardw.yang@linux.intel.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
migration/ram.c | 22 ++++++++++++----------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/migration/ram.c b/migration/ram.c
index 889148dd..88ddd2bb 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -922,7 +922,7 @@ struct {
* false.
*/
-static int multifd_send_pages(void)
+static int multifd_send_pages(RAMState *rs)
{
int i;
static int next_channel;
@@ -954,6 +954,7 @@ static int multifd_send_pages(void)
multifd_send_state->pages = p->pages;
p->pages = pages;
transferred = ((uint64_t) pages->used) * TARGET_PAGE_SIZE + p->packet_len;
+ qemu_file_update_transfer(rs->f, transferred);
ram_counters.multifd_bytes += transferred;
ram_counters.transferred += transferred;;
qemu_mutex_unlock(&p->mutex);
@@ -962,7 +963,7 @@ static int multifd_send_pages(void)
return 1;
}
-static int multifd_queue_page(RAMBlock *block, ram_addr_t offset)
+static int multifd_queue_page(RAMState *rs, RAMBlock *block, ram_addr_t offset)
{
MultiFDPages_t *pages = multifd_send_state->pages;
@@ -981,12 +982,12 @@ static int multifd_queue_page(RAMBlock *block, ram_addr_t offset)
}
}
- if (multifd_send_pages() < 0) {
+ if (multifd_send_pages(rs) < 0) {
return -1;
}
if (pages->block != block) {
- return multifd_queue_page(block, offset);
+ return multifd_queue_page(rs, block, offset);
}
return 1;
@@ -1054,7 +1055,7 @@ void multifd_save_cleanup(void)
multifd_send_state = NULL;
}
-static void multifd_send_sync_main(void)
+static void multifd_send_sync_main(RAMState *rs)
{
int i;
@@ -1062,7 +1063,7 @@ static void multifd_send_sync_main(void)
return;
}
if (multifd_send_state->pages->used) {
- if (multifd_send_pages() < 0) {
+ if (multifd_send_pages(rs) < 0) {
error_report("%s: multifd_send_pages fail", __func__);
return;
}
@@ -1083,6 +1084,7 @@ static void multifd_send_sync_main(void)
p->packet_num = multifd_send_state->packet_num++;
p->flags |= MULTIFD_FLAG_SYNC;
p->pending_job++;
+ qemu_file_update_transfer(rs->f, p->packet_len);
qemu_mutex_unlock(&p->mutex);
qemu_sem_post(&p->sem);
}
@@ -2079,7 +2081,7 @@ static int ram_save_page(RAMState *rs, PageSearchStatus *pss, bool last_stage)
static int ram_save_multifd_page(RAMState *rs, RAMBlock *block,
ram_addr_t offset)
{
- if (multifd_queue_page(block, offset) < 0) {
+ if (multifd_queue_page(rs, block, offset) < 0) {
return -1;
}
ram_counters.normal++;
@@ -3482,7 +3484,7 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
ram_control_before_iterate(f, RAM_CONTROL_SETUP);
ram_control_after_iterate(f, RAM_CONTROL_SETUP);
- multifd_send_sync_main();
+ multifd_send_sync_main(*rsp);
qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
qemu_fflush(f);
@@ -3570,7 +3572,7 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
ram_control_after_iterate(f, RAM_CONTROL_ROUND);
out:
- multifd_send_sync_main();
+ multifd_send_sync_main(rs);
qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
qemu_fflush(f);
ram_counters.transferred += 8;
@@ -3629,7 +3631,7 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
rcu_read_unlock();
- multifd_send_sync_main();
+ multifd_send_sync_main(rs);
qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
qemu_fflush(f);
--
2.19.1

125
migration-always-initialise-ram_counters-for-a-new-m.patch Normal file
View File

@ -0,0 +1,125 @@
From af2aa4f553565ae6b2248204c154748f38ec4746 Mon Sep 17 00:00:00 2001
From: Ivan Ren <ivanren@tencent.com>
Date: Fri, 2 Aug 2019 18:18:41 +0800
Subject: [PATCH 01/10] migration: always initialise ram_counters for a new
migration
This patch fix a multifd migration bug in migration speed calculation, this
problem can be reproduced as follows:
1. start a vm and give a heavy memory write stress to prevent the vm be
successfully migrated to destination
2. begin a migration with multifd
3. migrate for a long time [actually, this can be measured by transferred bytes]
4. migrate cancel
5. begin a new migration with multifd, the migration will directly run into
migration_completion phase
Reason as follows:
Migration update bandwidth and s->threshold_size in function
migration_update_counters after BUFFER_DELAY time:
current_bytes = migration_total_bytes(s);
transferred = current_bytes - s->iteration_initial_bytes;
time_spent = current_time - s->iteration_start_time;
bandwidth = (double)transferred / time_spent;
s->threshold_size = bandwidth * s->parameters.downtime_limit;
In multifd migration, migration_total_bytes function return
qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes.
s->iteration_initial_bytes will be initialized to 0 at every new migration,
but ram_counters is a global variable, and history migration data will be
accumulated. So if the ram_counters.multifd_bytes is big enough, it may lead
pending_size >= s->threshold_size become false in migration_iteration_run
after the first migration_update_counters.
Change-Id: Ib153d8676a5b82650bfb1156060e09f0d29f3ac6
Signed-off-by: Ivan Ren <ivanren@tencent.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Reviewed-by: Wei Yang <richardw.yang@linux.intel.com>
Suggested-by: Wei Yang <richardw.yang@linux.intel.com>
Message-Id: <1564741121-1840-1-git-send-email-ivanren@tencent.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
migration/migration.c | 25 +++++++++++++++++++------
migration/savevm.c | 1 +
2 files changed, 20 insertions(+), 6 deletions(-)
diff --git a/migration/migration.c b/migration/migration.c
index 8a607fe1..bea9b1d7 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -1908,6 +1908,11 @@ static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
}
migrate_init(s);
+ /*
+ * set ram_counters memory to zero for a
+ * new migration
+ */
+ memset(&ram_counters, 0, sizeof(ram_counters));
return true;
}
@@ -3025,6 +3030,17 @@ static void migration_calculate_complete(MigrationState *s)
}
}
+static void update_iteration_initial_status(MigrationState *s)
+{
+ /*
+ * Update these three fields at the same time to avoid mismatch info lead
+ * wrong speed calculation.
+ */
+ s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+ s->iteration_initial_bytes = migration_total_bytes(s);
+ s->iteration_initial_pages = ram_get_total_transferred_pages();
+}
+
static void migration_update_counters(MigrationState *s,
int64_t current_time)
{
@@ -3060,9 +3076,7 @@ static void migration_update_counters(MigrationState *s,
qemu_file_reset_rate_limit(s->to_dst_file);
- s->iteration_start_time = current_time;
- s->iteration_initial_bytes = current_bytes;
- s->iteration_initial_pages = ram_get_total_transferred_pages();
+ update_iteration_initial_status(s);
trace_migrate_transferred(transferred, time_spent,
bandwidth, s->threshold_size);
@@ -3186,7 +3200,7 @@ static void *migration_thread(void *opaque)
rcu_register_thread();
object_ref(OBJECT(s));
- s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+ update_iteration_initial_status(s);
qemu_savevm_state_header(s->to_dst_file);
@@ -3251,8 +3265,7 @@ static void *migration_thread(void *opaque)
* the local variables. This is important to avoid
* breaking transferred_bytes and bandwidth calculation
*/
- s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
- s->iteration_initial_bytes = 0;
+ update_iteration_initial_status(s);
}
current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
diff --git a/migration/savevm.c b/migration/savevm.c
index 79ed44d4..480c511b 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -1424,6 +1424,7 @@ static int qemu_savevm_state(QEMUFile *f, Error **errp)
}
migrate_init(ms);
+ memset(&ram_counters, 0, sizeof(ram_counters));
ms->to_dst_file = f;
qemu_mutex_unlock_iothread();
--
2.19.1

64
migration-multifd-fix-destroyed-mutex-access-in-term.patch Normal file
View File

@ -0,0 +1,64 @@
From 34d797aa134a33c1d67ca85d9d9f996d58162276 Mon Sep 17 00:00:00 2001
From: Jiahui Cen <cenjiahui@huawei.com>
Date: Wed, 23 Oct 2019 11:47:37 +0800
Subject: [PATCH 09/10] migration/multifd: fix destroyed mutex access in
terminating multifd threads
One multifd will lock all the other multifds' IOChannel mutex to inform them
to quit by setting p->quit or shutting down p->c. In this senario, if some
multifds had already been terminated and multifd_load_cleanup/multifd_save_cleanup
had destroyed their mutex, it could cause destroyed mutex access when trying
lock their mutex.
Here is the coredump stack:
#0 0x00007f81a2794437 in raise () from /usr/lib64/libc.so.6
#1 0x00007f81a2795b28 in abort () from /usr/lib64/libc.so.6
#2 0x00007f81a278d1b6 in __assert_fail_base () from /usr/lib64/libc.so.6
#3 0x00007f81a278d262 in __assert_fail () from /usr/lib64/libc.so.6
#4 0x000055eb1bfadbd3 in qemu_mutex_lock_impl (mutex=0x55eb1e2d1988, file=<optimized out>, line=<optimized out>) at util/qemu-thread-posix.c:64
#5 0x000055eb1bb4564a in multifd_send_terminate_threads (err=<optimized out>) at migration/ram.c:1015
#6 0x000055eb1bb4bb7f in multifd_send_thread (opaque=0x55eb1e2d19f8) at migration/ram.c:1171
#7 0x000055eb1bfad628 in qemu_thread_start (args=0x55eb1e170450) at util/qemu-thread-posix.c:502
#8 0x00007f81a2b36df5 in start_thread () from /usr/lib64/libpthread.so.0
#9 0x00007f81a286048d in clone () from /usr/lib64/libc.so.6
To fix it up, let's destroy the mutex after all the other multifd threads had
been terminated.
Change-Id: I4124d43e8558ba302052bdc53fdae7cfcf9d8687
Signed-off-by: Jiahui Cen <cenjiahui@huawei.com>
Signed-off-by: Ying Fang <fangying1@huawei.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
---
migration/ram.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/migration/ram.c b/migration/ram.c
index 029f1cdf..d7d2d5ec 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -1033,6 +1033,10 @@ void multifd_save_cleanup(void)
if (p->running) {
qemu_thread_join(&p->thread);
}
+ }
+ for (i = 0; i < migrate_multifd_channels(); i++) {
+ MultiFDSendParams *p = &multifd_send_state->params[i];
+
socket_send_channel_destroy(p->c);
p->c = NULL;
qemu_mutex_destroy(&p->mutex);
@@ -1306,6 +1310,10 @@ int multifd_load_cleanup(Error **errp)
qemu_sem_post(&p->sem_sync);
qemu_thread_join(&p->thread);
}
+ }
+ for (i = 0; i < migrate_multifd_channels(); i++) {
+ MultiFDRecvParams *p = &multifd_recv_state->params[i];
+
object_unref(OBJECT(p->c));
p->c = NULL;
qemu_mutex_destroy(&p->mutex);
--
2.19.1

62
migration-multifd-fix-nullptr-access-in-multifd_send.patch Normal file
View File

@ -0,0 +1,62 @@
From 6a08ee257a95d9f2514bd995e90ddf46d3f78b41 Mon Sep 17 00:00:00 2001
From: Zheng Chuan <zhengchuan@huawei.com>
Date: Tue, 21 Apr 2020 19:49:26 +0800
Subject: [PATCH 10/10] migration/multifd: fix nullptr access in
multifd_send_terminate_threads
If the multifd_send_threads is not created when migration is failed,
multifd_save_cleanup would be called twice. In this senario, the
multifd_send_state is accessed after it has been released, the result
is that the source VM is crashing down.
Here is the coredump stack:
Program received signal SIGSEGV, Segmentation fault.
0x00005629333a78ef in multifd_send_terminate_threads (err=err@entry=0x0) at migration/ram.c:1012
1012 MultiFDSendParams *p = &multifd_send_state->params[i];
#0 0x00005629333a78ef in multifd_send_terminate_threads (err=err@entry=0x0) at migration/ram.c:1012
#1 0x00005629333ab8a9 in multifd_save_cleanup () at migration/ram.c:1028
#2 0x00005629333abaea in multifd_new_send_channel_async (task=0x562935450e70, opaque=<optimized out>) at migration/ram.c:1202
#3 0x000056293373a562 in qio_task_complete (task=task@entry=0x562935450e70) at io/task.c:196
#4 0x000056293373a6e0 in qio_task_thread_result (opaque=0x562935450e70) at io/task.c:111
#5 0x00007f475d4d75a7 in g_idle_dispatch () from /usr/lib64/libglib-2.0.so.0
#6 0x00007f475d4da9a9 in g_main_context_dispatch () from /usr/lib64/libglib-2.0.so.0
#7 0x0000562933785b33 in glib_pollfds_poll () at util/main-loop.c:219
#8 os_host_main_loop_wait (timeout=<optimized out>) at util/main-loop.c:242
#9 main_loop_wait (nonblocking=nonblocking@entry=0) at util/main-loop.c:518
#10 0x00005629334c5acf in main_loop () at vl.c:1810
#11 0x000056293334d7bb in main (argc=<optimized out>, argv=<optimized out>, envp=<optimized out>) at vl.c:4471
If the multifd_send_threads is not created when migration is failed.
In this senario, we don't call multifd_save_cleanup in multifd_new_send_channel_async.
Change-Id: I7441efe2ed542054ecd2a4da8146e2652824b452
Signed-off-by: Zhimin Feng <fengzhimin1@huawei.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
---
migration/ram.c | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/migration/ram.c b/migration/ram.c
index d7d2d5ec..1858d66c 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -1205,7 +1205,15 @@ static void multifd_new_send_channel_async(QIOTask *task, gpointer opaque)
if (qio_task_propagate_error(task, &local_err)) {
migrate_set_error(migrate_get_current(), local_err);
- multifd_save_cleanup();
+ /* Error happen, we need to tell who pay attention to me */
+ qemu_sem_post(&multifd_send_state->channels_ready);
+ qemu_sem_post(&p->sem_sync);
+ /*
+ * Although multifd_send_thread is not created, but main migration
+ * thread neet to judge whether it is running, so we need to mark
+ * its status.
+ */
+ p->quit = true;
} else {
p->c = QIO_CHANNEL(sioc);
qio_channel_set_delay(p->c, false);
--
2.19.1

75
migration-multifd-fix-nullptr-access-in-terminating-m.patch Normal file
View File

@ -0,0 +1,75 @@
From d9a847f0982fcca6f63031215065c346fcc27bbc Mon Sep 17 00:00:00 2001
From: Zheng Chuan <zhengchuan@huawei.com>
Date: Fri, 24 Apr 2020 11:58:33 +0800
Subject: [PATCH 06/10] migration/multifd: fix nullptr access in terminating
multifd threads
One multifd channel will shutdown all the other multifd's IOChannel when it
fails to receive an IOChannel. In this senario, if some multifds had not
received its IOChannel yet, it would try to shutdown its IOChannel which could
cause nullptr access at qio_channel_shutdown.
Here is the coredump stack:
#0 object_get_class (obj=obj@entry=0x0) at qom/object.c:908
#1 0x00005563fdbb8f4a in qio_channel_shutdown (ioc=0x0, how=QIO_CHANNEL_SHUTDOWN_BOTH, errp=0x0) at io/channel.c:355
#2 0x00005563fd7b4c5f in multifd_recv_terminate_threads (err=<optimized out>) at migration/ram.c:1280
#3 0x00005563fd7bc019 in multifd_recv_new_channel (ioc=ioc@entry=0x556400255610, errp=errp@entry=0x7ffec07dce00) at migration/ram.c:1478
#4 0x00005563fda82177 in migration_ioc_process_incoming (ioc=ioc@entry=0x556400255610, errp=errp@entry=0x7ffec07dce30) at migration/migration.c:605
#5 0x00005563fda8567d in migration_channel_process_incoming (ioc=0x556400255610) at migration/channel.c:44
#6 0x00005563fda83ee0 in socket_accept_incoming_migration (listener=0x5563fff6b920, cioc=0x556400255610, opaque=<optimized out>) at migration/socket
.c:166
#7 0x00005563fdbc25cd in qio_net_listener_channel_func (ioc=<optimized out>, condition=<optimized out>, opaque=<optimized out>) at io/net-listener.c:54
#8 0x00007f895b6fe9a9 in g_main_context_dispatch () from /usr/lib64/libglib-2.0.so.0
#9 0x00005563fdc18136 in glib_pollfds_poll () at util/main-loop.c:218
#10 0x00005563fdc181b5 in os_host_main_loop_wait (timeout=1000000000) at util/main-loop.c:241
#11 0x00005563fdc183a2 in main_loop_wait (nonblocking=nonblocking@entry=0) at util/main-loop.c:517
#12 0x00005563fd8edb37 in main_loop () at vl.c:1791
#13 0x00005563fd74fd45 in main (argc=<optimized out>, argv=<optimized out>, envp=<optimized out>) at vl.c:4473
To fix it up, let's check p->c before calling qio_channel_shutdown.
Change-Id: Ib36c1b3d866a3ad92d1460512df840cfb8736ab6
Signed-off-by: Jiahui Cen <cenjiahui@huawei.com>
Signed-off-by: Ying Fang <fangying1@huawei.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>
---
migration/ram.c | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/migration/ram.c b/migration/ram.c
index 51811c2d..756a525f 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -1112,6 +1112,7 @@ static void *multifd_send_thread(void *opaque)
rcu_register_thread();
if (multifd_send_initial_packet(p, &local_err) < 0) {
+ ret = -1;
goto out;
}
/* initial packet */
@@ -1178,9 +1179,7 @@ out:
* who pay attention to me.
*/
if (ret != 0) {
- if (flags & MULTIFD_FLAG_SYNC) {
- qemu_sem_post(&p->sem_sync);
- }
+ qemu_sem_post(&p->sem_sync);
qemu_sem_post(&multifd_send_state->channels_ready);
}
@@ -1279,7 +1278,9 @@ static void multifd_recv_terminate_threads(Error *err)
- normal quit, i.e. everything went fine, just finished
- error quit: We close the channels so the channel threads
finish the qio_channel_read_all_eof() */
- qio_channel_shutdown(p->c, QIO_CHANNEL_SHUTDOWN_BOTH, NULL);
+ if (p->c) {
+ qio_channel_shutdown(p->c, QIO_CHANNEL_SHUTDOWN_BOTH, NULL);
+ }
qemu_mutex_unlock(&p->mutex);
}
}
--
2.19.1

294
migration-multifd-fix-potential-wrong-acception-orde.patch Normal file
View File

@ -0,0 +1,294 @@
From 71f3e496c128b46f803cc4776154b02a5e505cb2 Mon Sep 17 00:00:00 2001
From: Zheng Chuan <zhengchuan@huawei.com>
Date: Wed, 22 Apr 2020 13:45:39 +0800
Subject: [PATCH] migration/multifd: fix potential wrong acception order of
IOChannel
Multifd assumes the migration thread IOChannel is always established before
the multifd IOChannels, but this assumption will be broken in many situations
like network packet loss.
For example:
Step1: Source (migration thread IOChannel) --SYN--> Destination
Step2: Source (migration thread IOChannel) <--SYNACK Destination
Step3: Source (migration thread IOChannel, lost) --ACK-->X Destination
Step4: Source (multifd IOChannel) --SYN--> Destination
Step5: Source (multifd IOChannel) <--SYNACK Destination
Step6: Source (multifd IOChannel, ESTABLISHED) --ACK--> Destination
Step7: Destination accepts multifd IOChannel
Step8: Source (migration thread IOChannel, ESTABLISHED) -ACK,DATA-> Destination
Step9: Destination accepts migration thread IOChannel
The above situation can be reproduced by creating a weak network environment,
such as "tc qdisc add dev eth0 root netem loss 50%". The wrong acception order
will cause magic check failure and thus lead to migration failure.
This patch fixes this issue by sending a migration IOChannel initial packet with
a unique id when using multifd migration. Since the multifd IOChannels will also
send initial packets, the destination can judge whether the processing IOChannel
belongs to multifd by checking the id in the initial packet. This mechanism can
ensure that different IOChannels will go to correct branches in our test.
Change-Id: I63d1c32c7b66063bd6a3c5e7d63500555bd148b9
Signed-off-by: Jiahui Cen <cenjiahui@huawei.com>
Signed-off-by: Ying Fang <fangying1@huawei.com>
diff --git a/migration/channel.c b/migration/channel.c
index 20e4c8e2..74621814 100644
--- a/migration/channel.c
+++ b/migration/channel.c
@@ -82,6 +82,15 @@ void migration_channel_connect(MigrationState *s,
return;
}
} else {
+ if (migrate_use_multifd()) {
+ /* multifd migration cannot distinguish migration IOChannel
+ * from multifd IOChannels, so we need to send an initial packet
+ * to show it is migration IOChannel
+ */
+ migration_send_initial_packet(ioc,
+ migrate_multifd_channels(),
+ &error);
+ }
QEMUFile *f = qemu_fopen_channel_output(ioc);
qemu_mutex_lock(&s->qemu_file_lock);
diff --git a/migration/migration.c b/migration/migration.c
index 114c33a1..8f2fc2b4 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -517,12 +517,6 @@ static void migration_incoming_setup(QEMUFile *f)
{
MigrationIncomingState *mis = migration_incoming_get_current();
- if (multifd_load_setup() != 0) {
- /* We haven't been able to create multifd threads
- nothing better to do */
- exit(EXIT_FAILURE);
- }
-
if (!mis->from_src_file) {
mis->from_src_file = f;
}
@@ -580,36 +574,41 @@ void migration_fd_process_incoming(QEMUFile *f)
void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
{
MigrationIncomingState *mis = migration_incoming_get_current();
- bool start_migration;
-
- if (!mis->from_src_file) {
- /* The first connection (multifd may have multiple) */
- QEMUFile *f = qemu_fopen_channel_input(ioc);
+ Error *local_err = NULL;
+ int id = 0;
- /* If it's a recovery, we're done */
- if (postcopy_try_recover(f)) {
- return;
- }
+ if (migrate_use_multifd()) {
+ id = migration_recv_initial_packet(ioc, &local_err);
+ }
+ if (!migrate_use_multifd() || id == migrate_multifd_channels()) {
+ if (!mis->from_src_file) {
+ /* The migration connection (multifd may have multiple) */
+ QEMUFile *f = qemu_fopen_channel_input(ioc);
- migration_incoming_setup(f);
+ /* If it's a recovery, we're done */
+ if (postcopy_try_recover(f)) {
+ return;
+ }
- /*
- * Common migration only needs one channel, so we can start
- * right now. Multifd needs more than one channel, we wait.
- */
- start_migration = !migrate_use_multifd();
- } else {
- Error *local_err = NULL;
+ migration_incoming_setup(f);
+ }
+ } else if (id >= 0) {
/* Multiple connections */
assert(migrate_use_multifd());
- start_migration = multifd_recv_new_channel(ioc, &local_err);
+ multifd_recv_new_channel(ioc, id, &local_err);
if (local_err) {
error_propagate(errp, local_err);
return;
}
+ } else {
+ /* Bad connections */
+ multifd_recv_terminate_threads(local_err);
+ error_propagate(errp, local_err);
+ return;
}
- if (start_migration) {
+ /* Once we have all the channels we need, we can start migration */
+ if (migration_has_all_channels()) {
migration_incoming_process();
}
}
diff --git a/migration/migration.h b/migration/migration.h
index 1fdd7b21..feb34430 100644
--- a/migration/migration.h
+++ b/migration/migration.h
@@ -339,4 +339,7 @@ int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque);
void migration_make_urgent_request(void);
void migration_consume_urgent_request(void);
+int migration_send_initial_packet(QIOChannel *c, uint8_t id, Error **errp);
+int migration_recv_initial_packet(QIOChannel *c, Error **errp);
+
#endif
diff --git a/migration/ram.c b/migration/ram.c
index 756a525f..029f1cdf 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -593,7 +593,7 @@ typedef struct {
uint8_t id;
uint8_t unused1[7]; /* Reserved for future use */
uint64_t unused2[4]; /* Reserved for future use */
-} __attribute__((packed)) MultiFDInit_t;
+} __attribute__((packed)) MigrationInit_t;
typedef struct {
uint32_t magic;
@@ -702,26 +702,26 @@ typedef struct {
QemuSemaphore sem_sync;
} MultiFDRecvParams;
-static int multifd_send_initial_packet(MultiFDSendParams *p, Error **errp)
+int migration_send_initial_packet(QIOChannel *c, uint8_t id, Error **errp)
{
- MultiFDInit_t msg;
+ MigrationInit_t msg;
int ret;
msg.magic = cpu_to_be32(MULTIFD_MAGIC);
msg.version = cpu_to_be32(MULTIFD_VERSION);
- msg.id = p->id;
+ msg.id = id;
memcpy(msg.uuid, &qemu_uuid.data, sizeof(msg.uuid));
- ret = qio_channel_write_all(p->c, (char *)&msg, sizeof(msg), errp);
+ ret = qio_channel_write_all(c, (char *)&msg, sizeof(msg), errp);
if (ret != 0) {
return -1;
}
return 0;
}
-static int multifd_recv_initial_packet(QIOChannel *c, Error **errp)
+int migration_recv_initial_packet(QIOChannel *c, Error **errp)
{
- MultiFDInit_t msg;
+ MigrationInit_t msg;
int ret;
ret = qio_channel_read_all(c, (char *)&msg, sizeof(msg), errp);
@@ -756,8 +756,8 @@ static int multifd_recv_initial_packet(QIOChannel *c, Error **errp)
}
if (msg.id > migrate_multifd_channels()) {
- error_setg(errp, "multifd: received channel version %d "
- "expected %d", msg.version, MULTIFD_VERSION);
+ error_setg(errp, "multifd: received channel id %d "
+ "expected [0-%d]", msg.id, migrate_multifd_channels());
return -1;
}
@@ -1111,7 +1111,7 @@ static void *multifd_send_thread(void *opaque)
trace_multifd_send_thread_start(p->id);
rcu_register_thread();
- if (multifd_send_initial_packet(p, &local_err) < 0) {
+ if (migration_send_initial_packet(p->c, p->id, &local_err) < 0) {
ret = -1;
goto out;
}
@@ -1255,7 +1255,7 @@ struct {
uint64_t packet_num;
} *multifd_recv_state;
-static void multifd_recv_terminate_threads(Error *err)
+void multifd_recv_terminate_threads(Error *err)
{
int i;
@@ -1470,21 +1470,10 @@ bool multifd_recv_all_channels_created(void)
* - Return false and do not set @errp when correctly receiving the current one;
* - Return false and set @errp when failing to receive the current channel.
*/
-bool multifd_recv_new_channel(QIOChannel *ioc, Error **errp)
+void multifd_recv_new_channel(QIOChannel *ioc, int id, Error **errp)
{
MultiFDRecvParams *p;
Error *local_err = NULL;
- int id;
-
- id = multifd_recv_initial_packet(ioc, &local_err);
- if (id < 0) {
- multifd_recv_terminate_threads(local_err);
- error_propagate_prepend(errp, local_err,
- "failed to receive packet"
- " via multifd channel %d: ",
- atomic_read(&multifd_recv_state->count));
- return false;
- }
p = &multifd_recv_state->params[id];
if (p->c != NULL) {
@@ -1492,7 +1481,7 @@ bool multifd_recv_new_channel(QIOChannel *ioc, Error **errp)
id);
multifd_recv_terminate_threads(local_err);
error_propagate(errp, local_err);
- return false;
+ return;
}
p->c = ioc;
object_ref(OBJECT(ioc));
@@ -1503,8 +1492,6 @@ bool multifd_recv_new_channel(QIOChannel *ioc, Error **errp)
qemu_thread_create(&p->thread, p->name, multifd_recv_thread, p,
QEMU_THREAD_JOINABLE);
atomic_inc(&multifd_recv_state->count);
- return atomic_read(&multifd_recv_state->count) ==
- migrate_multifd_channels();
}
/**
diff --git a/migration/ram.h b/migration/ram.h
index bd0eee79..a788ff0e 100644
--- a/migration/ram.h
+++ b/migration/ram.h
@@ -46,7 +46,8 @@ void multifd_save_cleanup(void);
int multifd_load_setup(void);
int multifd_load_cleanup(Error **errp);
bool multifd_recv_all_channels_created(void);
-bool multifd_recv_new_channel(QIOChannel *ioc, Error **errp);
+void multifd_recv_new_channel(QIOChannel *ioc, int id, Error **errp);
+void multifd_recv_terminate_threads(Error *err);
uint64_t ram_pagesize_summary(void);
int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len);
diff --git a/migration/socket.c b/migration/socket.c
index 98efdc02..bc0960c6 100644
--- a/migration/socket.c
+++ b/migration/socket.c
@@ -181,6 +181,12 @@ static void socket_start_incoming_migration(SocketAddress *saddr,
qio_net_listener_set_name(listener, "migration-socket-listener");
+ if (multifd_load_setup() != 0) {
+ /* We haven't been able to create multifd threads
+ nothing better to do */
+ exit(EXIT_FAILURE);
+ }
+
if (qio_net_listener_open_sync(listener, saddr, errp) < 0) {
object_unref(OBJECT(listener));
return;
--
2.23.0

35
migration-update-ram_counters-for-multifd-sync-packe.patch Normal file
View File

@ -0,0 +1,35 @@
From e93040851d683f1f7750acfa0e862b4405678f24 Mon Sep 17 00:00:00 2001
From: Zheng Chuan <zhengchuan@huawei.com>
Date: Fri, 24 Apr 2020 11:50:41 +0800
Subject: [PATCH 04/10] migration: update ram_counters for multifd sync packet
Multifd sync will send MULTIFD_FLAG_SYNC flag info to destination, add
these bytes to ram_counters record.
Change-Id: I885166f412f58e74de40ea6ffec1c35e82ae4619
Signed-off-by: Ivan Ren <ivanren@tencent.com>
Suggested-by: Wei Yang <richardw.yang@linux.intel.com>
Message-Id: <1564464816-21804-4-git-send-email-ivanren@tencent.com>
Reviewed-by: Juan Quintela <quintela@redhat.com>
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
migration/ram.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/migration/ram.c b/migration/ram.c
index 88ddd2bb..c75716bb 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -1085,6 +1085,10 @@ static void multifd_send_sync_main(RAMState *rs)
p->flags |= MULTIFD_FLAG_SYNC;
p->pending_job++;
qemu_file_update_transfer(rs->f, p->packet_len);
+ ram_counters.multifd_bytes += p->packet_len;
+ ram_counters.transferred += p->packet_len;
+ ram_counters.multifd_bytes += p->packet_len;
+ ram_counters.transferred += p->packet_len;
qemu_mutex_unlock(&p->mutex);
qemu_sem_post(&p->sem);
}
--
2.19.1

14
qemu.spec
View File

@ -147,7 +147,16 @@ Patch0134: arm-virt-acpi-Extend-cpufreq-to-support-max_cpus.patch
Patch0135: arm-virt-Pre-sizing-MADT-GICC-PPTT-GICv3-and-Pre-par.patch Patch0135: arm-virt-Pre-sizing-MADT-GICC-PPTT-GICv3-and-Pre-par.patch
Patch0136: arm-virt-Add-some-sanity-checks-in-cpu_pre_plug-hook.patch Patch0136: arm-virt-Add-some-sanity-checks-in-cpu_pre_plug-hook.patch
Patch0137: arm-virt-Start-up-CPU-hot-plug.patch Patch0137: arm-virt-Start-up-CPU-hot-plug.patch
Patch0138: migration-always-initialise-ram_counters-for-a-new-m.patch
Patch0139: migration-add-qemu_file_update_transfer-interface.patch
Patch0140: migration-add-speed-limit-for-multifd-migration.patch
Patch0141: migration-update-ram_counters-for-multifd-sync-packe.patch
Patch0142: migration-Make-global-sem_sync-semaphore-by-channel.patch
Patch0143: migration-multifd-fix-nullptr-access-in-terminating-m.patch
Patch0144: migration-Maybe-VM-is-paused-when-migration-is-cance.patch
Patch0145: migration-multifd-fix-potential-wrong-acception-orde.patch
Patch0146: migration-multifd-fix-destroyed-mutex-access-in-term.patch
Patch0147: migration-multifd-fix-nullptr-access-in-multifd_send.patch
BuildRequires: flex BuildRequires: flex
BuildRequires: bison BuildRequires: bison
@ -493,6 +502,9 @@ getent passwd qemu >/dev/null || \
%endif %endif
%changelog %changelog
* Fri Apr 24 2020 Huawei Technologies Co., Ltd. <zhengchuan@huawei.com>
- migration: backport migration patches from upstream
* Fri Apr 24 2020 Huawei Technologies Co., Ltd. <zhukeqian1@huawei.com> * Fri Apr 24 2020 Huawei Technologies Co., Ltd. <zhukeqian1@huawei.com>
- arm/virt: Add CPU hotplug support - arm/virt: Add CPU hotplug support