diff --git a/migration-Make-global-sem_sync-semaphore-by-channel.patch b/migration-Make-global-sem_sync-semaphore-by-channel.patch new file mode 100644 index 0000000..d9dbab2 --- /dev/null +++ b/migration-Make-global-sem_sync-semaphore-by-channel.patch @@ -0,0 +1,100 @@ +From 8c3794d709eefdae777477bef7ff3511d55bf418 Mon Sep 17 00:00:00 2001 +From: Juan Quintela +Date: Wed, 14 Aug 2019 04:02:14 +0200 +Subject: [PATCH 05/10] migration: Make global sem_sync semaphore by channel + +This makes easy to debug things because when you want for all threads +to arrive at that semaphore, you know which one your are waiting for. + +Change-Id: I533af8cdc68f619b68eff8e4e573c4de371a3954 +Signed-off-by: Juan Quintela +Message-Id: <20190814020218.1868-3-quintela@redhat.com> +Reviewed-by: Dr. David Alan Gilbert +Signed-off-by: Dr. David Alan Gilbert +--- + migration/ram.c | 14 +++++++------- + 1 file changed, 7 insertions(+), 7 deletions(-) + +diff --git a/migration/ram.c b/migration/ram.c +index c75716bb..51811c2d 100644 +--- a/migration/ram.c ++++ b/migration/ram.c +@@ -661,6 +661,8 @@ typedef struct { + uint64_t num_packets; + /* pages sent through this channel */ + uint64_t num_pages; ++ /* syncs main thread and channels */ ++ QemuSemaphore sem_sync; + } MultiFDSendParams; + + typedef struct { +@@ -896,8 +898,6 @@ struct { + MultiFDSendParams *params; + /* array of pages to sent */ + MultiFDPages_t *pages; +- /* syncs main thread and channels */ +- QemuSemaphore sem_sync; + /* global number of generated multifd packets */ + uint64_t packet_num; + /* send channels ready */ +@@ -1037,6 +1037,7 @@ void multifd_save_cleanup(void) + p->c = NULL; + qemu_mutex_destroy(&p->mutex); + qemu_sem_destroy(&p->sem); ++ qemu_sem_destroy(&p->sem_sync); + g_free(p->name); + p->name = NULL; + multifd_pages_clear(p->pages); +@@ -1046,7 +1047,6 @@ void multifd_save_cleanup(void) + p->packet = NULL; + } + qemu_sem_destroy(&multifd_send_state->channels_ready); +- qemu_sem_destroy(&multifd_send_state->sem_sync); + g_free(multifd_send_state->params); + multifd_send_state->params = NULL; + multifd_pages_clear(multifd_send_state->pages); +@@ -1096,7 +1096,7 @@ static void multifd_send_sync_main(RAMState *rs) + MultiFDSendParams *p = &multifd_send_state->params[i]; + + trace_multifd_send_sync_main_wait(p->id); +- qemu_sem_wait(&multifd_send_state->sem_sync); ++ qemu_sem_wait(&p->sem_sync); + } + trace_multifd_send_sync_main(multifd_send_state->packet_num); + } +@@ -1156,7 +1156,7 @@ static void *multifd_send_thread(void *opaque) + qemu_mutex_unlock(&p->mutex); + + if (flags & MULTIFD_FLAG_SYNC) { +- qemu_sem_post(&multifd_send_state->sem_sync); ++ qemu_sem_post(&p->sem_sync); + } + qemu_sem_post(&multifd_send_state->channels_ready); + } else if (p->quit) { +@@ -1179,7 +1179,7 @@ out: + */ + if (ret != 0) { + if (flags & MULTIFD_FLAG_SYNC) { +- qemu_sem_post(&multifd_send_state->sem_sync); ++ qemu_sem_post(&p->sem_sync); + } + qemu_sem_post(&multifd_send_state->channels_ready); + } +@@ -1225,7 +1225,6 @@ int multifd_save_setup(void) + multifd_send_state = g_malloc0(sizeof(*multifd_send_state)); + multifd_send_state->params = g_new0(MultiFDSendParams, thread_count); + multifd_send_state->pages = multifd_pages_init(page_count); +- qemu_sem_init(&multifd_send_state->sem_sync, 0); + qemu_sem_init(&multifd_send_state->channels_ready, 0); + + for (i = 0; i < thread_count; i++) { +@@ -1233,6 +1232,7 @@ int multifd_save_setup(void) + + qemu_mutex_init(&p->mutex); + qemu_sem_init(&p->sem, 0); ++ qemu_sem_init(&p->sem_sync, 0); + p->quit = false; + p->pending_job = 0; + p->id = i; +-- +2.19.1 diff --git a/migration-Maybe-VM-is-paused-when-migration-is-cance.patch b/migration-Maybe-VM-is-paused-when-migration-is-cance.patch new file mode 100644 index 0000000..6c918f3 --- /dev/null +++ b/migration-Maybe-VM-is-paused-when-migration-is-cance.patch @@ -0,0 +1,57 @@ +From 5e99e1329fa52dce8ab784a960e64a3e19b429aa Mon Sep 17 00:00:00 2001 +From: Zhimin Feng +Date: Tue, 14 Jan 2020 17:43:09 +0800 +Subject: [PATCH 07/10] migration: Maybe VM is paused when migration is + cancelled + +If the migration is cancelled when it is in the completion phase, +the migration state is set to MIGRATION_STATUS_CANCELLING. +The VM maybe wait for the 'pause_sem' semaphore in migration_maybe_pause +function, so that VM always is paused. + +Change-Id: Ib2f2f42ee1edbb14da269ee19ba1fe16dd363822 +Reported-by: Euler Robot +Signed-off-by: Zhimin Feng +Reviewed-by: Juan Quintela +Signed-off-by: Juan Quintela +--- + migration/migration.c | 24 ++++++++++++++++-------- + 1 file changed, 16 insertions(+), 8 deletions(-) + +diff --git a/migration/migration.c b/migration/migration.c +index bea9b1d7..114c33a1 100644 +--- a/migration/migration.c ++++ b/migration/migration.c +@@ -2731,14 +2731,22 @@ static int migration_maybe_pause(MigrationState *s, + /* This block intentionally left blank */ + } + +- qemu_mutex_unlock_iothread(); +- migrate_set_state(&s->state, *current_active_state, +- MIGRATION_STATUS_PRE_SWITCHOVER); +- qemu_sem_wait(&s->pause_sem); +- migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER, +- new_state); +- *current_active_state = new_state; +- qemu_mutex_lock_iothread(); ++ /* ++ * If the migration is cancelled when it is in the completion phase, ++ * the migration state is set to MIGRATION_STATUS_CANCELLING. ++ * So we don't need to wait a semaphore, otherwise we would always ++ * wait for the 'pause_sem' semaphore. ++ */ ++ if (s->state != MIGRATION_STATUS_CANCELLING) { ++ qemu_mutex_unlock_iothread(); ++ migrate_set_state(&s->state, *current_active_state, ++ MIGRATION_STATUS_PRE_SWITCHOVER); ++ qemu_sem_wait(&s->pause_sem); ++ migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER, ++ new_state); ++ *current_active_state = new_state; ++ qemu_mutex_lock_iothread(); ++ } + + return s->state == new_state ? 0 : -EINVAL; + } +-- +2.19.1 diff --git a/migration-add-qemu_file_update_transfer-interface.patch b/migration-add-qemu_file_update_transfer-interface.patch new file mode 100644 index 0000000..4222fd0 --- /dev/null +++ b/migration-add-qemu_file_update_transfer-interface.patch @@ -0,0 +1,50 @@ +From 7572495245a437da717e6829a9ce852cc3f229c9 Mon Sep 17 00:00:00 2001 +From: Zheng Chuan +Date: Mon, 20 Apr 2020 15:13:47 +0800 +Subject: [PATCH 02/10] migration: add qemu_file_update_transfer interface + +Add qemu_file_update_transfer for just update bytes_xfer for speed +limitation. This will be used for further migration feature such as +multifd migration. + +Change-Id: I969aa15305c961254b6fb9805b0ed2d65826cc5d +Signed-off-by: Ivan Ren +Reviewed-by: Wei Yang +Reviewed-by: Juan Quintela +Message-Id: <1564464816-21804-2-git-send-email-ivanren@tencent.com> +Signed-off-by: Dr. David Alan Gilbert +--- + migration/qemu-file.c | 5 +++++ + migration/qemu-file.h | 1 + + 2 files changed, 6 insertions(+) + +diff --git a/migration/qemu-file.c b/migration/qemu-file.c +index 04315855..18f48052 100644 +--- a/migration/qemu-file.c ++++ b/migration/qemu-file.c +@@ -615,6 +615,11 @@ void qemu_file_reset_rate_limit(QEMUFile *f) + f->bytes_xfer = 0; + } + ++void qemu_file_update_transfer(QEMUFile *f, int64_t len) ++{ ++ f->bytes_xfer += len; ++} ++ + void qemu_put_be16(QEMUFile *f, unsigned int v) + { + qemu_put_byte(f, v >> 8); +diff --git a/migration/qemu-file.h b/migration/qemu-file.h +index 13baf896..5de9fa2e 100644 +--- a/migration/qemu-file.h ++++ b/migration/qemu-file.h +@@ -147,6 +147,7 @@ int qemu_peek_byte(QEMUFile *f, int offset); + void qemu_file_skip(QEMUFile *f, int size); + void qemu_update_position(QEMUFile *f, size_t size); + void qemu_file_reset_rate_limit(QEMUFile *f); ++void qemu_file_update_transfer(QEMUFile *f, int64_t len); + void qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate); + int64_t qemu_file_get_rate_limit(QEMUFile *f); + void qemu_file_set_error(QEMUFile *f, int ret); +-- +2.19.1 diff --git a/migration-add-speed-limit-for-multifd-migration.patch b/migration-add-speed-limit-for-multifd-migration.patch new file mode 100644 index 0000000..690d9c9 --- /dev/null +++ b/migration-add-speed-limit-for-multifd-migration.patch @@ -0,0 +1,127 @@ +From bc5780480db9e38699df0b4697e60a9f36258dc4 Mon Sep 17 00:00:00 2001 +From: Ivan Ren +Date: Tue, 30 Jul 2019 13:33:35 +0800 +Subject: [PATCH 03/10] migration: add speed limit for multifd migration + +Limit the speed of multifd migration through common speed limitation +qemu file. + +Change-Id: Id2abfc7ea85679bd53130a43043cc70179a52e87 +Signed-off-by: Ivan Ren +Message-Id: <1564464816-21804-3-git-send-email-ivanren@tencent.com> +Reviewed-by: Wei Yang +Reviewed-by: Juan Quintela +Signed-off-by: Dr. David Alan Gilbert +--- + migration/ram.c | 22 ++++++++++++---------- + 1 file changed, 12 insertions(+), 10 deletions(-) + +diff --git a/migration/ram.c b/migration/ram.c +index 889148dd..88ddd2bb 100644 +--- a/migration/ram.c ++++ b/migration/ram.c +@@ -922,7 +922,7 @@ struct { + * false. + */ + +-static int multifd_send_pages(void) ++static int multifd_send_pages(RAMState *rs) + { + int i; + static int next_channel; +@@ -954,6 +954,7 @@ static int multifd_send_pages(void) + multifd_send_state->pages = p->pages; + p->pages = pages; + transferred = ((uint64_t) pages->used) * TARGET_PAGE_SIZE + p->packet_len; ++ qemu_file_update_transfer(rs->f, transferred); + ram_counters.multifd_bytes += transferred; + ram_counters.transferred += transferred;; + qemu_mutex_unlock(&p->mutex); +@@ -962,7 +963,7 @@ static int multifd_send_pages(void) + return 1; + } + +-static int multifd_queue_page(RAMBlock *block, ram_addr_t offset) ++static int multifd_queue_page(RAMState *rs, RAMBlock *block, ram_addr_t offset) + { + MultiFDPages_t *pages = multifd_send_state->pages; + +@@ -981,12 +982,12 @@ static int multifd_queue_page(RAMBlock *block, ram_addr_t offset) + } + } + +- if (multifd_send_pages() < 0) { ++ if (multifd_send_pages(rs) < 0) { + return -1; + } + + if (pages->block != block) { +- return multifd_queue_page(block, offset); ++ return multifd_queue_page(rs, block, offset); + } + + return 1; +@@ -1054,7 +1055,7 @@ void multifd_save_cleanup(void) + multifd_send_state = NULL; + } + +-static void multifd_send_sync_main(void) ++static void multifd_send_sync_main(RAMState *rs) + { + int i; + +@@ -1062,7 +1063,7 @@ static void multifd_send_sync_main(void) + return; + } + if (multifd_send_state->pages->used) { +- if (multifd_send_pages() < 0) { ++ if (multifd_send_pages(rs) < 0) { + error_report("%s: multifd_send_pages fail", __func__); + return; + } +@@ -1083,6 +1084,7 @@ static void multifd_send_sync_main(void) + p->packet_num = multifd_send_state->packet_num++; + p->flags |= MULTIFD_FLAG_SYNC; + p->pending_job++; ++ qemu_file_update_transfer(rs->f, p->packet_len); + qemu_mutex_unlock(&p->mutex); + qemu_sem_post(&p->sem); + } +@@ -2079,7 +2081,7 @@ static int ram_save_page(RAMState *rs, PageSearchStatus *pss, bool last_stage) + static int ram_save_multifd_page(RAMState *rs, RAMBlock *block, + ram_addr_t offset) + { +- if (multifd_queue_page(block, offset) < 0) { ++ if (multifd_queue_page(rs, block, offset) < 0) { + return -1; + } + ram_counters.normal++; +@@ -3482,7 +3484,7 @@ static int ram_save_setup(QEMUFile *f, void *opaque) + ram_control_before_iterate(f, RAM_CONTROL_SETUP); + ram_control_after_iterate(f, RAM_CONTROL_SETUP); + +- multifd_send_sync_main(); ++ multifd_send_sync_main(*rsp); + qemu_put_be64(f, RAM_SAVE_FLAG_EOS); + qemu_fflush(f); + +@@ -3570,7 +3572,7 @@ static int ram_save_iterate(QEMUFile *f, void *opaque) + ram_control_after_iterate(f, RAM_CONTROL_ROUND); + + out: +- multifd_send_sync_main(); ++ multifd_send_sync_main(rs); + qemu_put_be64(f, RAM_SAVE_FLAG_EOS); + qemu_fflush(f); + ram_counters.transferred += 8; +@@ -3629,7 +3631,7 @@ static int ram_save_complete(QEMUFile *f, void *opaque) + + rcu_read_unlock(); + +- multifd_send_sync_main(); ++ multifd_send_sync_main(rs); + qemu_put_be64(f, RAM_SAVE_FLAG_EOS); + qemu_fflush(f); + +-- +2.19.1 diff --git a/migration-always-initialise-ram_counters-for-a-new-m.patch b/migration-always-initialise-ram_counters-for-a-new-m.patch new file mode 100644 index 0000000..ccd0db9 --- /dev/null +++ b/migration-always-initialise-ram_counters-for-a-new-m.patch @@ -0,0 +1,125 @@ +From af2aa4f553565ae6b2248204c154748f38ec4746 Mon Sep 17 00:00:00 2001 +From: Ivan Ren +Date: Fri, 2 Aug 2019 18:18:41 +0800 +Subject: [PATCH 01/10] migration: always initialise ram_counters for a new + migration + +This patch fix a multifd migration bug in migration speed calculation, this +problem can be reproduced as follows: +1. start a vm and give a heavy memory write stress to prevent the vm be + successfully migrated to destination +2. begin a migration with multifd +3. migrate for a long time [actually, this can be measured by transferred bytes] +4. migrate cancel +5. begin a new migration with multifd, the migration will directly run into + migration_completion phase + +Reason as follows: + +Migration update bandwidth and s->threshold_size in function +migration_update_counters after BUFFER_DELAY time: + + current_bytes = migration_total_bytes(s); + transferred = current_bytes - s->iteration_initial_bytes; + time_spent = current_time - s->iteration_start_time; + bandwidth = (double)transferred / time_spent; + s->threshold_size = bandwidth * s->parameters.downtime_limit; + +In multifd migration, migration_total_bytes function return +qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes. +s->iteration_initial_bytes will be initialized to 0 at every new migration, +but ram_counters is a global variable, and history migration data will be +accumulated. So if the ram_counters.multifd_bytes is big enough, it may lead +pending_size >= s->threshold_size become false in migration_iteration_run +after the first migration_update_counters. + +Change-Id: Ib153d8676a5b82650bfb1156060e09f0d29f3ac6 +Signed-off-by: Ivan Ren +Reviewed-by: Juan Quintela +Reviewed-by: Wei Yang +Suggested-by: Wei Yang +Message-Id: <1564741121-1840-1-git-send-email-ivanren@tencent.com> +Signed-off-by: Dr. David Alan Gilbert +--- + migration/migration.c | 25 +++++++++++++++++++------ + migration/savevm.c | 1 + + 2 files changed, 20 insertions(+), 6 deletions(-) + +diff --git a/migration/migration.c b/migration/migration.c +index 8a607fe1..bea9b1d7 100644 +--- a/migration/migration.c ++++ b/migration/migration.c +@@ -1908,6 +1908,11 @@ static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc, + } + + migrate_init(s); ++ /* ++ * set ram_counters memory to zero for a ++ * new migration ++ */ ++ memset(&ram_counters, 0, sizeof(ram_counters)); + + return true; + } +@@ -3025,6 +3030,17 @@ static void migration_calculate_complete(MigrationState *s) + } + } + ++static void update_iteration_initial_status(MigrationState *s) ++{ ++ /* ++ * Update these three fields at the same time to avoid mismatch info lead ++ * wrong speed calculation. ++ */ ++ s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); ++ s->iteration_initial_bytes = migration_total_bytes(s); ++ s->iteration_initial_pages = ram_get_total_transferred_pages(); ++} ++ + static void migration_update_counters(MigrationState *s, + int64_t current_time) + { +@@ -3060,9 +3076,7 @@ static void migration_update_counters(MigrationState *s, + + qemu_file_reset_rate_limit(s->to_dst_file); + +- s->iteration_start_time = current_time; +- s->iteration_initial_bytes = current_bytes; +- s->iteration_initial_pages = ram_get_total_transferred_pages(); ++ update_iteration_initial_status(s); + + trace_migrate_transferred(transferred, time_spent, + bandwidth, s->threshold_size); +@@ -3186,7 +3200,7 @@ static void *migration_thread(void *opaque) + rcu_register_thread(); + + object_ref(OBJECT(s)); +- s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); ++ update_iteration_initial_status(s); + + qemu_savevm_state_header(s->to_dst_file); + +@@ -3251,8 +3265,7 @@ static void *migration_thread(void *opaque) + * the local variables. This is important to avoid + * breaking transferred_bytes and bandwidth calculation + */ +- s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); +- s->iteration_initial_bytes = 0; ++ update_iteration_initial_status(s); + } + + current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); +diff --git a/migration/savevm.c b/migration/savevm.c +index 79ed44d4..480c511b 100644 +--- a/migration/savevm.c ++++ b/migration/savevm.c +@@ -1424,6 +1424,7 @@ static int qemu_savevm_state(QEMUFile *f, Error **errp) + } + + migrate_init(ms); ++ memset(&ram_counters, 0, sizeof(ram_counters)); + ms->to_dst_file = f; + + qemu_mutex_unlock_iothread(); +-- +2.19.1 diff --git a/migration-multifd-fix-destroyed-mutex-access-in-term.patch b/migration-multifd-fix-destroyed-mutex-access-in-term.patch new file mode 100644 index 0000000..a927ea5 --- /dev/null +++ b/migration-multifd-fix-destroyed-mutex-access-in-term.patch @@ -0,0 +1,64 @@ +From 34d797aa134a33c1d67ca85d9d9f996d58162276 Mon Sep 17 00:00:00 2001 +From: Jiahui Cen +Date: Wed, 23 Oct 2019 11:47:37 +0800 +Subject: [PATCH 09/10] migration/multifd: fix destroyed mutex access in + terminating multifd threads + +One multifd will lock all the other multifds' IOChannel mutex to inform them +to quit by setting p->quit or shutting down p->c. In this senario, if some +multifds had already been terminated and multifd_load_cleanup/multifd_save_cleanup +had destroyed their mutex, it could cause destroyed mutex access when trying +lock their mutex. + +Here is the coredump stack: + #0 0x00007f81a2794437 in raise () from /usr/lib64/libc.so.6 + #1 0x00007f81a2795b28 in abort () from /usr/lib64/libc.so.6 + #2 0x00007f81a278d1b6 in __assert_fail_base () from /usr/lib64/libc.so.6 + #3 0x00007f81a278d262 in __assert_fail () from /usr/lib64/libc.so.6 + #4 0x000055eb1bfadbd3 in qemu_mutex_lock_impl (mutex=0x55eb1e2d1988, file=, line=) at util/qemu-thread-posix.c:64 + #5 0x000055eb1bb4564a in multifd_send_terminate_threads (err=) at migration/ram.c:1015 + #6 0x000055eb1bb4bb7f in multifd_send_thread (opaque=0x55eb1e2d19f8) at migration/ram.c:1171 + #7 0x000055eb1bfad628 in qemu_thread_start (args=0x55eb1e170450) at util/qemu-thread-posix.c:502 + #8 0x00007f81a2b36df5 in start_thread () from /usr/lib64/libpthread.so.0 + #9 0x00007f81a286048d in clone () from /usr/lib64/libc.so.6 + +To fix it up, let's destroy the mutex after all the other multifd threads had +been terminated. + +Change-Id: I4124d43e8558ba302052bdc53fdae7cfcf9d8687 +Signed-off-by: Jiahui Cen +Signed-off-by: Ying Fang +Reviewed-by: Juan Quintela +Signed-off-by: Juan Quintela +--- + migration/ram.c | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/migration/ram.c b/migration/ram.c +index 029f1cdf..d7d2d5ec 100644 +--- a/migration/ram.c ++++ b/migration/ram.c +@@ -1033,6 +1033,10 @@ void multifd_save_cleanup(void) + if (p->running) { + qemu_thread_join(&p->thread); + } ++ } ++ for (i = 0; i < migrate_multifd_channels(); i++) { ++ MultiFDSendParams *p = &multifd_send_state->params[i]; ++ + socket_send_channel_destroy(p->c); + p->c = NULL; + qemu_mutex_destroy(&p->mutex); +@@ -1306,6 +1310,10 @@ int multifd_load_cleanup(Error **errp) + qemu_sem_post(&p->sem_sync); + qemu_thread_join(&p->thread); + } ++ } ++ for (i = 0; i < migrate_multifd_channels(); i++) { ++ MultiFDRecvParams *p = &multifd_recv_state->params[i]; ++ + object_unref(OBJECT(p->c)); + p->c = NULL; + qemu_mutex_destroy(&p->mutex); +-- +2.19.1 diff --git a/migration-multifd-fix-nullptr-access-in-multifd_send.patch b/migration-multifd-fix-nullptr-access-in-multifd_send.patch new file mode 100644 index 0000000..f2d278a --- /dev/null +++ b/migration-multifd-fix-nullptr-access-in-multifd_send.patch @@ -0,0 +1,62 @@ +From 6a08ee257a95d9f2514bd995e90ddf46d3f78b41 Mon Sep 17 00:00:00 2001 +From: Zheng Chuan +Date: Tue, 21 Apr 2020 19:49:26 +0800 +Subject: [PATCH 10/10] migration/multifd: fix nullptr access in + multifd_send_terminate_threads + +If the multifd_send_threads is not created when migration is failed, +multifd_save_cleanup would be called twice. In this senario, the +multifd_send_state is accessed after it has been released, the result +is that the source VM is crashing down. + +Here is the coredump stack: + Program received signal SIGSEGV, Segmentation fault. + 0x00005629333a78ef in multifd_send_terminate_threads (err=err@entry=0x0) at migration/ram.c:1012 + 1012 MultiFDSendParams *p = &multifd_send_state->params[i]; + #0 0x00005629333a78ef in multifd_send_terminate_threads (err=err@entry=0x0) at migration/ram.c:1012 + #1 0x00005629333ab8a9 in multifd_save_cleanup () at migration/ram.c:1028 + #2 0x00005629333abaea in multifd_new_send_channel_async (task=0x562935450e70, opaque=) at migration/ram.c:1202 + #3 0x000056293373a562 in qio_task_complete (task=task@entry=0x562935450e70) at io/task.c:196 + #4 0x000056293373a6e0 in qio_task_thread_result (opaque=0x562935450e70) at io/task.c:111 + #5 0x00007f475d4d75a7 in g_idle_dispatch () from /usr/lib64/libglib-2.0.so.0 + #6 0x00007f475d4da9a9 in g_main_context_dispatch () from /usr/lib64/libglib-2.0.so.0 + #7 0x0000562933785b33 in glib_pollfds_poll () at util/main-loop.c:219 + #8 os_host_main_loop_wait (timeout=) at util/main-loop.c:242 + #9 main_loop_wait (nonblocking=nonblocking@entry=0) at util/main-loop.c:518 + #10 0x00005629334c5acf in main_loop () at vl.c:1810 + #11 0x000056293334d7bb in main (argc=, argv=, envp=) at vl.c:4471 + +If the multifd_send_threads is not created when migration is failed. +In this senario, we don't call multifd_save_cleanup in multifd_new_send_channel_async. + +Change-Id: I7441efe2ed542054ecd2a4da8146e2652824b452 +Signed-off-by: Zhimin Feng +Reviewed-by: Juan Quintela +Signed-off-by: Juan Quintela +--- + migration/ram.c | 10 +++++++++- + 1 file changed, 9 insertions(+), 1 deletion(-) + +diff --git a/migration/ram.c b/migration/ram.c +index d7d2d5ec..1858d66c 100644 +--- a/migration/ram.c ++++ b/migration/ram.c +@@ -1205,7 +1205,15 @@ static void multifd_new_send_channel_async(QIOTask *task, gpointer opaque) + + if (qio_task_propagate_error(task, &local_err)) { + migrate_set_error(migrate_get_current(), local_err); +- multifd_save_cleanup(); ++ /* Error happen, we need to tell who pay attention to me */ ++ qemu_sem_post(&multifd_send_state->channels_ready); ++ qemu_sem_post(&p->sem_sync); ++ /* ++ * Although multifd_send_thread is not created, but main migration ++ * thread neet to judge whether it is running, so we need to mark ++ * its status. ++ */ ++ p->quit = true; + } else { + p->c = QIO_CHANNEL(sioc); + qio_channel_set_delay(p->c, false); +-- +2.19.1 diff --git a/migration-multifd-fix-nullptr-access-in-terminating-m.patch b/migration-multifd-fix-nullptr-access-in-terminating-m.patch new file mode 100644 index 0000000..d403b28 --- /dev/null +++ b/migration-multifd-fix-nullptr-access-in-terminating-m.patch @@ -0,0 +1,75 @@ +From d9a847f0982fcca6f63031215065c346fcc27bbc Mon Sep 17 00:00:00 2001 +From: Zheng Chuan +Date: Fri, 24 Apr 2020 11:58:33 +0800 +Subject: [PATCH 06/10] migration/multifd: fix nullptr access in terminating + multifd threads + +One multifd channel will shutdown all the other multifd's IOChannel when it +fails to receive an IOChannel. In this senario, if some multifds had not +received its IOChannel yet, it would try to shutdown its IOChannel which could +cause nullptr access at qio_channel_shutdown. + +Here is the coredump stack: + #0 object_get_class (obj=obj@entry=0x0) at qom/object.c:908 + #1 0x00005563fdbb8f4a in qio_channel_shutdown (ioc=0x0, how=QIO_CHANNEL_SHUTDOWN_BOTH, errp=0x0) at io/channel.c:355 + #2 0x00005563fd7b4c5f in multifd_recv_terminate_threads (err=) at migration/ram.c:1280 + #3 0x00005563fd7bc019 in multifd_recv_new_channel (ioc=ioc@entry=0x556400255610, errp=errp@entry=0x7ffec07dce00) at migration/ram.c:1478 + #4 0x00005563fda82177 in migration_ioc_process_incoming (ioc=ioc@entry=0x556400255610, errp=errp@entry=0x7ffec07dce30) at migration/migration.c:605 + #5 0x00005563fda8567d in migration_channel_process_incoming (ioc=0x556400255610) at migration/channel.c:44 + #6 0x00005563fda83ee0 in socket_accept_incoming_migration (listener=0x5563fff6b920, cioc=0x556400255610, opaque=) at migration/socket +.c:166 + #7 0x00005563fdbc25cd in qio_net_listener_channel_func (ioc=, condition=, opaque=) at io/net-listener.c:54 + #8 0x00007f895b6fe9a9 in g_main_context_dispatch () from /usr/lib64/libglib-2.0.so.0 + #9 0x00005563fdc18136 in glib_pollfds_poll () at util/main-loop.c:218 + #10 0x00005563fdc181b5 in os_host_main_loop_wait (timeout=1000000000) at util/main-loop.c:241 + #11 0x00005563fdc183a2 in main_loop_wait (nonblocking=nonblocking@entry=0) at util/main-loop.c:517 + #12 0x00005563fd8edb37 in main_loop () at vl.c:1791 + #13 0x00005563fd74fd45 in main (argc=, argv=, envp=) at vl.c:4473 + +To fix it up, let's check p->c before calling qio_channel_shutdown. + +Change-Id: Ib36c1b3d866a3ad92d1460512df840cfb8736ab6 +Signed-off-by: Jiahui Cen +Signed-off-by: Ying Fang +Reviewed-by: Juan Quintela +Signed-off-by: Juan Quintela +--- + migration/ram.c | 9 +++++---- + 1 file changed, 5 insertions(+), 4 deletions(-) + +diff --git a/migration/ram.c b/migration/ram.c +index 51811c2d..756a525f 100644 +--- a/migration/ram.c ++++ b/migration/ram.c +@@ -1112,6 +1112,7 @@ static void *multifd_send_thread(void *opaque) + rcu_register_thread(); + + if (multifd_send_initial_packet(p, &local_err) < 0) { ++ ret = -1; + goto out; + } + /* initial packet */ +@@ -1178,9 +1179,7 @@ out: + * who pay attention to me. + */ + if (ret != 0) { +- if (flags & MULTIFD_FLAG_SYNC) { +- qemu_sem_post(&p->sem_sync); +- } ++ qemu_sem_post(&p->sem_sync); + qemu_sem_post(&multifd_send_state->channels_ready); + } + +@@ -1279,7 +1278,9 @@ static void multifd_recv_terminate_threads(Error *err) + - normal quit, i.e. everything went fine, just finished + - error quit: We close the channels so the channel threads + finish the qio_channel_read_all_eof() */ +- qio_channel_shutdown(p->c, QIO_CHANNEL_SHUTDOWN_BOTH, NULL); ++ if (p->c) { ++ qio_channel_shutdown(p->c, QIO_CHANNEL_SHUTDOWN_BOTH, NULL); ++ } + qemu_mutex_unlock(&p->mutex); + } + } +-- +2.19.1 diff --git a/migration-multifd-fix-potential-wrong-acception-orde.patch b/migration-multifd-fix-potential-wrong-acception-orde.patch new file mode 100644 index 0000000..c550203 --- /dev/null +++ b/migration-multifd-fix-potential-wrong-acception-orde.patch @@ -0,0 +1,294 @@ +From 71f3e496c128b46f803cc4776154b02a5e505cb2 Mon Sep 17 00:00:00 2001 +From: Zheng Chuan +Date: Wed, 22 Apr 2020 13:45:39 +0800 +Subject: [PATCH] migration/multifd: fix potential wrong acception order of + IOChannel + +Multifd assumes the migration thread IOChannel is always established before +the multifd IOChannels, but this assumption will be broken in many situations +like network packet loss. + +For example: +Step1: Source (migration thread IOChannel) --SYN--> Destination +Step2: Source (migration thread IOChannel) <--SYNACK Destination +Step3: Source (migration thread IOChannel, lost) --ACK-->X Destination +Step4: Source (multifd IOChannel) --SYN--> Destination +Step5: Source (multifd IOChannel) <--SYNACK Destination +Step6: Source (multifd IOChannel, ESTABLISHED) --ACK--> Destination +Step7: Destination accepts multifd IOChannel +Step8: Source (migration thread IOChannel, ESTABLISHED) -ACK,DATA-> Destination +Step9: Destination accepts migration thread IOChannel + +The above situation can be reproduced by creating a weak network environment, +such as "tc qdisc add dev eth0 root netem loss 50%". The wrong acception order +will cause magic check failure and thus lead to migration failure. + +This patch fixes this issue by sending a migration IOChannel initial packet with +a unique id when using multifd migration. Since the multifd IOChannels will also +send initial packets, the destination can judge whether the processing IOChannel +belongs to multifd by checking the id in the initial packet. This mechanism can +ensure that different IOChannels will go to correct branches in our test. + +Change-Id: I63d1c32c7b66063bd6a3c5e7d63500555bd148b9 +Signed-off-by: Jiahui Cen +Signed-off-by: Ying Fang + +diff --git a/migration/channel.c b/migration/channel.c +index 20e4c8e2..74621814 100644 +--- a/migration/channel.c ++++ b/migration/channel.c +@@ -82,6 +82,15 @@ void migration_channel_connect(MigrationState *s, + return; + } + } else { ++ if (migrate_use_multifd()) { ++ /* multifd migration cannot distinguish migration IOChannel ++ * from multifd IOChannels, so we need to send an initial packet ++ * to show it is migration IOChannel ++ */ ++ migration_send_initial_packet(ioc, ++ migrate_multifd_channels(), ++ &error); ++ } + QEMUFile *f = qemu_fopen_channel_output(ioc); + + qemu_mutex_lock(&s->qemu_file_lock); +diff --git a/migration/migration.c b/migration/migration.c +index 114c33a1..8f2fc2b4 100644 +--- a/migration/migration.c ++++ b/migration/migration.c +@@ -517,12 +517,6 @@ static void migration_incoming_setup(QEMUFile *f) + { + MigrationIncomingState *mis = migration_incoming_get_current(); + +- if (multifd_load_setup() != 0) { +- /* We haven't been able to create multifd threads +- nothing better to do */ +- exit(EXIT_FAILURE); +- } +- + if (!mis->from_src_file) { + mis->from_src_file = f; + } +@@ -580,36 +574,41 @@ void migration_fd_process_incoming(QEMUFile *f) + void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp) + { + MigrationIncomingState *mis = migration_incoming_get_current(); +- bool start_migration; +- +- if (!mis->from_src_file) { +- /* The first connection (multifd may have multiple) */ +- QEMUFile *f = qemu_fopen_channel_input(ioc); ++ Error *local_err = NULL; ++ int id = 0; + +- /* If it's a recovery, we're done */ +- if (postcopy_try_recover(f)) { +- return; +- } ++ if (migrate_use_multifd()) { ++ id = migration_recv_initial_packet(ioc, &local_err); ++ } ++ if (!migrate_use_multifd() || id == migrate_multifd_channels()) { ++ if (!mis->from_src_file) { ++ /* The migration connection (multifd may have multiple) */ ++ QEMUFile *f = qemu_fopen_channel_input(ioc); + +- migration_incoming_setup(f); ++ /* If it's a recovery, we're done */ ++ if (postcopy_try_recover(f)) { ++ return; ++ } + +- /* +- * Common migration only needs one channel, so we can start +- * right now. Multifd needs more than one channel, we wait. +- */ +- start_migration = !migrate_use_multifd(); +- } else { +- Error *local_err = NULL; ++ migration_incoming_setup(f); ++ } ++ } else if (id >= 0) { + /* Multiple connections */ + assert(migrate_use_multifd()); +- start_migration = multifd_recv_new_channel(ioc, &local_err); ++ multifd_recv_new_channel(ioc, id, &local_err); + if (local_err) { + error_propagate(errp, local_err); + return; + } ++ } else { ++ /* Bad connections */ ++ multifd_recv_terminate_threads(local_err); ++ error_propagate(errp, local_err); ++ return; + } + +- if (start_migration) { ++ /* Once we have all the channels we need, we can start migration */ ++ if (migration_has_all_channels()) { + migration_incoming_process(); + } + } +diff --git a/migration/migration.h b/migration/migration.h +index 1fdd7b21..feb34430 100644 +--- a/migration/migration.h ++++ b/migration/migration.h +@@ -339,4 +339,7 @@ int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque); + void migration_make_urgent_request(void); + void migration_consume_urgent_request(void); + ++int migration_send_initial_packet(QIOChannel *c, uint8_t id, Error **errp); ++int migration_recv_initial_packet(QIOChannel *c, Error **errp); ++ + #endif +diff --git a/migration/ram.c b/migration/ram.c +index 756a525f..029f1cdf 100644 +--- a/migration/ram.c ++++ b/migration/ram.c +@@ -593,7 +593,7 @@ typedef struct { + uint8_t id; + uint8_t unused1[7]; /* Reserved for future use */ + uint64_t unused2[4]; /* Reserved for future use */ +-} __attribute__((packed)) MultiFDInit_t; ++} __attribute__((packed)) MigrationInit_t; + + typedef struct { + uint32_t magic; +@@ -702,26 +702,26 @@ typedef struct { + QemuSemaphore sem_sync; + } MultiFDRecvParams; + +-static int multifd_send_initial_packet(MultiFDSendParams *p, Error **errp) ++int migration_send_initial_packet(QIOChannel *c, uint8_t id, Error **errp) + { +- MultiFDInit_t msg; ++ MigrationInit_t msg; + int ret; + + msg.magic = cpu_to_be32(MULTIFD_MAGIC); + msg.version = cpu_to_be32(MULTIFD_VERSION); +- msg.id = p->id; ++ msg.id = id; + memcpy(msg.uuid, &qemu_uuid.data, sizeof(msg.uuid)); + +- ret = qio_channel_write_all(p->c, (char *)&msg, sizeof(msg), errp); ++ ret = qio_channel_write_all(c, (char *)&msg, sizeof(msg), errp); + if (ret != 0) { + return -1; + } + return 0; + } + +-static int multifd_recv_initial_packet(QIOChannel *c, Error **errp) ++int migration_recv_initial_packet(QIOChannel *c, Error **errp) + { +- MultiFDInit_t msg; ++ MigrationInit_t msg; + int ret; + + ret = qio_channel_read_all(c, (char *)&msg, sizeof(msg), errp); +@@ -756,8 +756,8 @@ static int multifd_recv_initial_packet(QIOChannel *c, Error **errp) + } + + if (msg.id > migrate_multifd_channels()) { +- error_setg(errp, "multifd: received channel version %d " +- "expected %d", msg.version, MULTIFD_VERSION); ++ error_setg(errp, "multifd: received channel id %d " ++ "expected [0-%d]", msg.id, migrate_multifd_channels()); + return -1; + } + +@@ -1111,7 +1111,7 @@ static void *multifd_send_thread(void *opaque) + trace_multifd_send_thread_start(p->id); + rcu_register_thread(); + +- if (multifd_send_initial_packet(p, &local_err) < 0) { ++ if (migration_send_initial_packet(p->c, p->id, &local_err) < 0) { + ret = -1; + goto out; + } +@@ -1255,7 +1255,7 @@ struct { + uint64_t packet_num; + } *multifd_recv_state; + +-static void multifd_recv_terminate_threads(Error *err) ++void multifd_recv_terminate_threads(Error *err) + { + int i; + +@@ -1470,21 +1470,10 @@ bool multifd_recv_all_channels_created(void) + * - Return false and do not set @errp when correctly receiving the current one; + * - Return false and set @errp when failing to receive the current channel. + */ +-bool multifd_recv_new_channel(QIOChannel *ioc, Error **errp) ++void multifd_recv_new_channel(QIOChannel *ioc, int id, Error **errp) + { + MultiFDRecvParams *p; + Error *local_err = NULL; +- int id; +- +- id = multifd_recv_initial_packet(ioc, &local_err); +- if (id < 0) { +- multifd_recv_terminate_threads(local_err); +- error_propagate_prepend(errp, local_err, +- "failed to receive packet" +- " via multifd channel %d: ", +- atomic_read(&multifd_recv_state->count)); +- return false; +- } + + p = &multifd_recv_state->params[id]; + if (p->c != NULL) { +@@ -1492,7 +1481,7 @@ bool multifd_recv_new_channel(QIOChannel *ioc, Error **errp) + id); + multifd_recv_terminate_threads(local_err); + error_propagate(errp, local_err); +- return false; ++ return; + } + p->c = ioc; + object_ref(OBJECT(ioc)); +@@ -1503,8 +1492,6 @@ bool multifd_recv_new_channel(QIOChannel *ioc, Error **errp) + qemu_thread_create(&p->thread, p->name, multifd_recv_thread, p, + QEMU_THREAD_JOINABLE); + atomic_inc(&multifd_recv_state->count); +- return atomic_read(&multifd_recv_state->count) == +- migrate_multifd_channels(); + } + + /** +diff --git a/migration/ram.h b/migration/ram.h +index bd0eee79..a788ff0e 100644 +--- a/migration/ram.h ++++ b/migration/ram.h +@@ -46,7 +46,8 @@ void multifd_save_cleanup(void); + int multifd_load_setup(void); + int multifd_load_cleanup(Error **errp); + bool multifd_recv_all_channels_created(void); +-bool multifd_recv_new_channel(QIOChannel *ioc, Error **errp); ++void multifd_recv_new_channel(QIOChannel *ioc, int id, Error **errp); ++void multifd_recv_terminate_threads(Error *err); + + uint64_t ram_pagesize_summary(void); + int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len); +diff --git a/migration/socket.c b/migration/socket.c +index 98efdc02..bc0960c6 100644 +--- a/migration/socket.c ++++ b/migration/socket.c +@@ -181,6 +181,12 @@ static void socket_start_incoming_migration(SocketAddress *saddr, + + qio_net_listener_set_name(listener, "migration-socket-listener"); + ++ if (multifd_load_setup() != 0) { ++ /* We haven't been able to create multifd threads ++ nothing better to do */ ++ exit(EXIT_FAILURE); ++ } ++ + if (qio_net_listener_open_sync(listener, saddr, errp) < 0) { + object_unref(OBJECT(listener)); + return; +-- +2.23.0 diff --git a/migration-update-ram_counters-for-multifd-sync-packe.patch b/migration-update-ram_counters-for-multifd-sync-packe.patch new file mode 100644 index 0000000..8383804 --- /dev/null +++ b/migration-update-ram_counters-for-multifd-sync-packe.patch @@ -0,0 +1,35 @@ +From e93040851d683f1f7750acfa0e862b4405678f24 Mon Sep 17 00:00:00 2001 +From: Zheng Chuan +Date: Fri, 24 Apr 2020 11:50:41 +0800 +Subject: [PATCH 04/10] migration: update ram_counters for multifd sync packet + +Multifd sync will send MULTIFD_FLAG_SYNC flag info to destination, add +these bytes to ram_counters record. + +Change-Id: I885166f412f58e74de40ea6ffec1c35e82ae4619 +Signed-off-by: Ivan Ren +Suggested-by: Wei Yang +Message-Id: <1564464816-21804-4-git-send-email-ivanren@tencent.com> +Reviewed-by: Juan Quintela +Signed-off-by: Dr. David Alan Gilbert +--- + migration/ram.c | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/migration/ram.c b/migration/ram.c +index 88ddd2bb..c75716bb 100644 +--- a/migration/ram.c ++++ b/migration/ram.c +@@ -1085,6 +1085,10 @@ static void multifd_send_sync_main(RAMState *rs) + p->flags |= MULTIFD_FLAG_SYNC; + p->pending_job++; + qemu_file_update_transfer(rs->f, p->packet_len); ++ ram_counters.multifd_bytes += p->packet_len; ++ ram_counters.transferred += p->packet_len; ++ ram_counters.multifd_bytes += p->packet_len; ++ ram_counters.transferred += p->packet_len; + qemu_mutex_unlock(&p->mutex); + qemu_sem_post(&p->sem); + } +-- +2.19.1 diff --git a/qemu.spec b/qemu.spec index 98f310e..7ca1463 100644 --- a/qemu.spec +++ b/qemu.spec @@ -147,7 +147,16 @@ Patch0134: arm-virt-acpi-Extend-cpufreq-to-support-max_cpus.patch Patch0135: arm-virt-Pre-sizing-MADT-GICC-PPTT-GICv3-and-Pre-par.patch Patch0136: arm-virt-Add-some-sanity-checks-in-cpu_pre_plug-hook.patch Patch0137: arm-virt-Start-up-CPU-hot-plug.patch - +Patch0138: migration-always-initialise-ram_counters-for-a-new-m.patch +Patch0139: migration-add-qemu_file_update_transfer-interface.patch +Patch0140: migration-add-speed-limit-for-multifd-migration.patch +Patch0141: migration-update-ram_counters-for-multifd-sync-packe.patch +Patch0142: migration-Make-global-sem_sync-semaphore-by-channel.patch +Patch0143: migration-multifd-fix-nullptr-access-in-terminating-m.patch +Patch0144: migration-Maybe-VM-is-paused-when-migration-is-cance.patch +Patch0145: migration-multifd-fix-potential-wrong-acception-orde.patch +Patch0146: migration-multifd-fix-destroyed-mutex-access-in-term.patch +Patch0147: migration-multifd-fix-nullptr-access-in-multifd_send.patch BuildRequires: flex BuildRequires: bison @@ -493,6 +502,9 @@ getent passwd qemu >/dev/null || \ %endif %changelog +* Fri Apr 24 2020 Huawei Technologies Co., Ltd. +- migration: backport migration patches from upstream + * Fri Apr 24 2020 Huawei Technologies Co., Ltd. - arm/virt: Add CPU hotplug support