From 06bb2d68ef70813167a633aa00779acf61c784b0 Mon Sep 17 00:00:00 2001
From: jiangdongxu <jiangdongxu1@huawei.com>
Date: Tue, 19 Dec 2023 20:18:03 +0800
Subject: [PATCH] vdpa: support vdpa device suspend/resume

commit a21603f7ecfa 'vhost: implement vhost_vdpa_device_suspend/resume'
only implement suspend and resume interface used for migration. The
current implementation still has bugs when suspend/resume a virtual
machine. Fix it.

Signed-off-by: jiangdongxu <jiangdongxu1@huawei.com>
---
 hw/virtio/vdpa-dev-mig.c     | 16 +++++++++++-----
 hw/virtio/vdpa-dev.c         |  8 +-------
 include/hw/virtio/vdpa-dev.h |  1 +
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/hw/virtio/vdpa-dev-mig.c b/hw/virtio/vdpa-dev-mig.c
index c71e71fd64..4a45821892 100644
--- a/hw/virtio/vdpa-dev-mig.c
+++ b/hw/virtio/vdpa-dev-mig.c
@@ -149,6 +149,7 @@ static int vhost_vdpa_device_suspend(VhostVdpaDevice *vdpa)
     }
 
     vdpa->started = false;
+    vdpa->suspended = true;
 
     ret = vhost_dev_suspend(&vdpa->dev, vdev, false);
     if (ret) {
@@ -171,6 +172,7 @@ set_guest_notifiers_fail:
     }
 
 suspend_fail:
+    vdpa->suspended = false;
     vdpa->started = true;
     return ret;
 }
@@ -207,6 +209,7 @@ static int vhost_vdpa_device_resume(VhostVdpaDevice *vdpa)
         goto err_guest_notifiers;
     }
     vdpa->started = true;
+    vdpa->suspended = false;
 
     /*
      * guest_notifier_mask/pending not used yet, so just unmask
@@ -247,7 +250,7 @@ static void vdpa_dev_vmstate_change(void *opaque, bool running, RunState state)
     MigrationIncomingState *mis = migration_incoming_get_current();
 
     if (!running) {
-        if (ms->state == RUN_STATE_PAUSED) {
+        if (ms->state == MIGRATION_STATUS_ACTIVE || state == RUN_STATE_PAUSED) {
             ret = vhost_vdpa_device_suspend(vdpa);
             if (ret) {
                 error_report("suspend vdpa device failed: %d\n", ret);
@@ -257,16 +260,19 @@ static void vdpa_dev_vmstate_change(void *opaque, bool running, RunState state)
             }
         }
     } else {
-        if (ms->state == RUN_STATE_RESTORE_VM) {
+        if (vdpa->suspended) {
             ret = vhost_vdpa_device_resume(vdpa);
             if (ret) {
-                error_report("migration dest resume device failed, abort!\n");
-                exit(EXIT_FAILURE);
+                error_report("vhost vdpa device resume failed: %d\n", ret);
             }
         }
 
         if (mis->state == RUN_STATE_RESTORE_VM) {
-            vhost_vdpa_call(hdev, VHOST_VDPA_RESUME, NULL);
+            ret = vhost_vdpa_call(hdev, VHOST_VDPA_RESUME, NULL);
+            if (ret) {
+                error_report("migration dest resume device failed: %d\n", ret);
+                exit(EXIT_FAILURE);
+            }
             /* post resume */
             mis->bh = qemu_bh_new(vdpa_dev_migration_handle_incoming_bh,
                                   hdev);
diff --git a/hw/virtio/vdpa-dev.c b/hw/virtio/vdpa-dev.c
index 143dadc88d..04d8e96a5d 100644
--- a/hw/virtio/vdpa-dev.c
+++ b/hw/virtio/vdpa-dev.c
@@ -315,7 +315,6 @@ static void vhost_vdpa_device_stop(VirtIODevice *vdev)
 static void vhost_vdpa_device_set_status(VirtIODevice *vdev, uint8_t status)
 {
     VhostVdpaDevice *s = VHOST_VDPA_DEVICE(vdev);
-    MigrationState *ms = migrate_get_current();
     bool should_start = virtio_device_started(vdev, status);
     Error *local_err = NULL;
     int ret;
@@ -324,12 +323,7 @@ static void vhost_vdpa_device_set_status(VirtIODevice *vdev, uint8_t status)
         should_start = false;
     }
 
-    if (s->started == should_start) {
-        return;
-    }
-
-    if (ms->state == RUN_STATE_PAUSED ||
-        ms->state == RUN_STATE_RESTORE_VM) {
+    if (s->started == should_start || s->suspended) {
         return;
     }
 
diff --git a/include/hw/virtio/vdpa-dev.h b/include/hw/virtio/vdpa-dev.h
index 20f50c76c6..60e9c3f3fe 100644
--- a/include/hw/virtio/vdpa-dev.h
+++ b/include/hw/virtio/vdpa-dev.h
@@ -37,6 +37,7 @@ struct VhostVdpaDevice {
     int config_size;
     uint16_t queue_size;
     bool started;
+    bool suspended;
     int (*post_init)(VhostVdpaDevice *v, Error **errp);
     VMChangeStateEntry *vmstate;
     Notifier migration_state;
-- 
2.27.0