qemu/bakcend-VirtCCA-resolve-hugepage-memory-waste-issue-.patch

From da6ee14de85b4e619eedfbe3a6cac3f09d948589 Mon Sep 17 00:00:00 2001
From: nonce <2774337358@qq.com>
Date: Thu, 23 Jan 2025 21:03:10 +0800
Subject: [PATCH] bakcend: VirtCCA:resolve hugepage memory waste issue in
 vhost-user scenario

VirtCCA is based on SWIOTLB to implement virtio and will only allocate
Bounce Buffer in the lower address range below 4GB. Therefore, the
backend hugepages memory allocated above 4GB will not be used, resulting
in significant waste.

New address space and memory region are added to manage the backend
hugepages memory corresponding to the GPA below 4GB, and there are
shared with the vhostuser backend.

Signed-off-by: nonce0_0 <2774337358@qq.com>
---
 backends/hostmem-file.c       | 85 +++++++++++++++++++++++++++++++++++
 hw/core/numa.c                | 20 +++++++++
 hw/virtio/vhost.c             |  8 +++-
 include/exec/address-spaces.h |  3 ++
 include/exec/cpu-common.h     |  1 +
 include/exec/memory.h         | 11 +++++
 system/physmem.c              | 17 +++++++
 system/vl.c                   |  9 ++++
 8 files changed, 153 insertions(+), 1 deletion(-)

diff --git a/backends/hostmem-file.c b/backends/hostmem-file.c
index 361d4a8103..891fe4ac4a 100644
--- a/backends/hostmem-file.c
+++ b/backends/hostmem-file.c
@@ -20,9 +20,13 @@
 #include "qom/object.h"
 #include "qapi/visitor.h"
 #include "qapi/qapi-visit-common.h"
+#include "sysemu/kvm.h"
+#include "exec/address-spaces.h"
 
 OBJECT_DECLARE_SIMPLE_TYPE(HostMemoryBackendFile, MEMORY_BACKEND_FILE)
 
+bool virtcca_shared_hugepage_mapped = false;
+uint64_t virtcca_cvm_ram_size = 0;
 
 struct HostMemoryBackendFile {
     HostMemoryBackend parent_obj;
@@ -36,6 +40,83 @@ struct HostMemoryBackendFile {
     OnOffAuto rom;
 };
 
+/* Parse the path of the hugepages memory file used for memory sharing */
+static int virtcca_parse_share_mem_path(char *src, char *dst)
+{
+    int ret = 0;
+    char src_copy[PATH_MAX];
+    char *token = NULL;
+    char *last_dir = NULL;
+    char *second_last_dir = NULL;
+    static const char delimiter[] = "/";
+
+    if (src == NULL || dst == NULL ||
+        strlen(src) == 0 || strlen(src) > PATH_MAX - 1) {
+        error_report("Invalid input: NULL pointer or invalid string length.");
+        return -1;
+    }
+
+    strcpy(src_copy, src);
+    token = strtok(src_copy, delimiter);
+
+    /* Iterate over the path segments to find the second-to-last directory */
+    while (token != NULL) {
+        second_last_dir = last_dir;
+        last_dir = token;
+        token = strtok(NULL, delimiter);
+    }
+
+    /* Check if the second-to-last directory is found */
+    if (second_last_dir == NULL) {
+        error_report("Invalid path: second-to-last directory not found.");
+        return -1;
+    }
+
+    /*
+     * Construct the share memory path by appending the extracted domain name
+     * to the hugepages memory filesystem prefix
+     */
+    ret = snprintf(dst, PATH_MAX, "/dev/hugepages/libvirt/qemu/%s",
+                   second_last_dir);
+
+    if (ret < 0 || ret >= PATH_MAX) {
+        error_report("Error: snprintf failed to construct the share mem path");
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Create a hugepage memory region in the virtcca scenario
+ * for sharing with process like vhost-user and others.
+ */
+static void
+virtcca_shared_backend_memory_alloc(char *mem_path, uint32_t ram_flags, Error **errp)
+{
+    char dst[PATH_MAX];
+    uint64_t size = virtcca_cvm_ram_size;
+
+    if (virtcca_parse_share_mem_path(mem_path, dst)) {
+        error_report("parse virtcca share memory path failed");
+        exit(1);
+    }
+    if (virtcca_cvm_ram_size >= VIRTCCA_SHARED_HUGEPAGE_MAX_SIZE) {
+        size = VIRTCCA_SHARED_HUGEPAGE_MAX_SIZE;
+    }
+
+    virtcca_shared_hugepage = g_new(MemoryRegion, 1);
+    memory_region_init_ram_from_file(virtcca_shared_hugepage, NULL,
+                                     "virtcca_shared_hugepage", size,
+                                     VIRTCCA_SHARED_HUGEPAGE_ALIGN,
+                                     ram_flags, dst, 0, errp);
+    if (*errp) {
+        error_reportf_err(*errp, "cannot init RamBlock for virtcca_shared_hugepage: ");
+        exit(1);
+    }
+    virtcca_shared_hugepage_mapped = true;
+}
+
 static void
 file_backend_memory_alloc(HostMemoryBackend *backend, Error **errp)
 {
@@ -90,6 +171,10 @@ file_backend_memory_alloc(HostMemoryBackend *backend, Error **errp)
                                      backend->size, fb->align, ram_flags,
                                      fb->mem_path, fb->offset, errp);
     g_free(name);
+
+    if (virtcca_cvm_enabled() && backend->share && !virtcca_shared_hugepage_mapped) {
+        virtcca_shared_backend_memory_alloc(fb->mem_path, ram_flags, errp);
+    }
 #endif
 }
 
diff --git a/hw/core/numa.c b/hw/core/numa.c
index f08956ddb0..e7c48dab61 100644
--- a/hw/core/numa.c
+++ b/hw/core/numa.c
@@ -42,6 +42,8 @@
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "qemu/cutils.h"
+#include "exec/address-spaces.h"
+#include "sysemu/kvm.h"
 
 QemuOptsList qemu_numa_opts = {
     .name = "numa",
@@ -641,6 +643,21 @@ static void numa_init_memdev_container(MachineState *ms, MemoryRegion *ram)
     }
 }
 
+/*
+ * Add virtcca_shared_hugepage as a sub-MR to the root MR of address space
+ * address_space_memory and address_space_virtcca_shared_memory.
+ */
+static void virtcca_shared_memory_configuration(MachineState *ms)
+{
+    MemoryRegion *alias_mr = g_new(MemoryRegion, 1);
+
+    memory_region_add_subregion_overlap(ms->ram, 0, virtcca_shared_hugepage, 1);
+    memory_region_init_alias(alias_mr, NULL, "alias-mr", virtcca_shared_hugepage,
+                             0, int128_get64(virtcca_shared_hugepage->size));
+    memory_region_add_subregion(address_space_virtcca_shared_memory.root,
+                                VIRTCCA_GPA_START, alias_mr);
+}
+
 void numa_complete_configuration(MachineState *ms)
 {
     int i;
@@ -711,6 +728,9 @@ void numa_complete_configuration(MachineState *ms)
             memory_region_init(ms->ram, OBJECT(ms), mc->default_ram_id,
                                ms->ram_size);
             numa_init_memdev_container(ms, ms->ram);
+            if (virtcca_cvm_enabled() && virtcca_shared_hugepage->ram_block) {
+                virtcca_shared_memory_configuration(ms);
+            }
         }
         /* QEMU needs at least all unique node pair distances to build
          * the whole NUMA distance table. QEMU treats the distance table
diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
index d29075aa04..8b95558013 100644
--- a/hw/virtio/vhost.c
+++ b/hw/virtio/vhost.c
@@ -30,6 +30,7 @@
 #include "sysemu/dma.h"
 #include "trace.h"
 #include "qapi/qapi-commands-migration.h"
+#include "sysemu/kvm.h"
 
 /* enabled until disconnected backend stabilizes */
 #define _VHOST_DEBUG 1
@@ -1616,7 +1617,12 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
     hdev->log_size = 0;
     hdev->log_enabled = false;
     hdev->started = false;
-    memory_listener_register(&hdev->memory_listener, &address_space_memory);
+    if (virtcca_cvm_enabled()) {
+        memory_listener_register(&hdev->memory_listener,
+                                 &address_space_virtcca_shared_memory);
+    } else {
+        memory_listener_register(&hdev->memory_listener, &address_space_memory);
+    }
     QLIST_INSERT_HEAD(&vhost_devices, hdev, entry);
 
     /*
diff --git a/include/exec/address-spaces.h b/include/exec/address-spaces.h
index 0d0aa61d68..4518b5da86 100644
--- a/include/exec/address-spaces.h
+++ b/include/exec/address-spaces.h
@@ -33,6 +33,9 @@ MemoryRegion *get_system_io(void);
 
 extern AddressSpace address_space_memory;
 extern AddressSpace address_space_io;
+extern AddressSpace address_space_virtcca_shared_memory;
+
+extern MemoryRegion *virtcca_shared_hugepage;
 
 #endif
 
diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h
index c7fd30d5b9..d21d9990ad 100644
--- a/include/exec/cpu-common.h
+++ b/include/exec/cpu-common.h
@@ -28,6 +28,7 @@ typedef uint64_t vaddr;
 
 void cpu_exec_init_all(void);
 void cpu_exec_step_atomic(CPUState *cpu);
+void virtcca_shared_memory_address_space_init(void);
 
 /* Using intptr_t ensures that qemu_*_page_mask is sign-extended even
  * when intptr_t is 32-bit and we are aligning a long long.
diff --git a/include/exec/memory.h b/include/exec/memory.h
index 542c9da918..33778f5c64 100644
--- a/include/exec/memory.h
+++ b/include/exec/memory.h
@@ -243,6 +243,17 @@ typedef struct IOMMUTLBEvent {
 /* RAM FD is opened read-only */
 #define RAM_READONLY_FD (1 << 11)
 
+/* The GPA range of the VirtCCA bounce buffer is from 1GB to 4GB. */
+#define VIRTCCA_SHARED_HUGEPAGE_MAX_SIZE 0xc0000000ULL
+
+/* The VirtCCA shared hugepage memory granularity is 1GB */
+#define VIRTCCA_SHARED_HUGEPAGE_ALIGN 0x40000000ULL
+
+/* The GPA starting address of the VirtCCA CVM is 1GB */
+#define VIRTCCA_GPA_START 0x40000000ULL
+
+extern uint64_t virtcca_cvm_ram_size;
+
 static inline void iommu_notifier_init(IOMMUNotifier *n, IOMMUNotify fn,
                                        IOMMUNotifierFlag flags,
                                        hwaddr start, hwaddr end,
diff --git a/system/physmem.c b/system/physmem.c
index 250f315bc8..8f4be2d131 100644
--- a/system/physmem.c
+++ b/system/physmem.c
@@ -89,9 +89,17 @@ RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
 
 static MemoryRegion *system_memory;
 static MemoryRegion *system_io;
+static MemoryRegion *virtcca_shared_memory;
+
+/*
+ * Serves as the sub-MR of the root MR (virtcca_shared_memory)
+ * and is associated with the RAMBlock.
+ */
+MemoryRegion *virtcca_shared_hugepage;
 
 AddressSpace address_space_io;
 AddressSpace address_space_memory;
+AddressSpace address_space_virtcca_shared_memory;
 
 static MemoryRegion io_mem_unassigned;
 
@@ -2586,6 +2594,15 @@ static void memory_map_init(void)
     address_space_init(&address_space_io, system_io, "I/O");
 }
 
+void virtcca_shared_memory_address_space_init(void)
+{
+    virtcca_shared_memory = g_malloc(sizeof(*virtcca_shared_memory));
+    memory_region_init(virtcca_shared_memory, NULL,
+                       "virtcca_shared_memory", UINT64_MAX);
+    address_space_init(&address_space_virtcca_shared_memory,
+                       virtcca_shared_memory, "virtcca_shared_memory");
+}
+
 MemoryRegion *get_system_memory(void)
 {
     return system_memory;
diff --git a/system/vl.c b/system/vl.c
index a1e5e68773..7c10cd1337 100644
--- a/system/vl.c
+++ b/system/vl.c
@@ -3784,6 +3784,15 @@ void qemu_init(int argc, char **argv)
     configure_accelerators(argv[0]);
     phase_advance(PHASE_ACCEL_CREATED);
 
+    /*
+     * Must run after kvm_init completes, as virtcca_cvm_enabled()
+     * depends on initialization performed in kvm_init.
+     */
+    if (virtcca_cvm_enabled()) {
+        virtcca_cvm_ram_size = current_machine->ram_size;
+        virtcca_shared_memory_address_space_init();
+    }
+
     /*
      * Beware, QOM objects created before this point miss global and
      * compat properties.
-- 
2.41.0.windows.1