319 lines
8.4 KiB
Diff
319 lines
8.4 KiB
Diff
From 762c6c0c4c8618fc95aa25696345e369681d0465 Mon Sep 17 00:00:00 2001
|
||
From: Yang Yanchao <yangyanchao6@huawei.com>
|
||
Date: Fri, 25 Aug 2023 18:39:54 +0800
|
||
Subject: [PATCH] use ioctl instead of syscall
|
||
|
||
---
|
||
Makefile.am | 3 ++-
|
||
include/gmem_common.h | 16 ++++++------
|
||
include/libgmem.h | 24 +++++++++++------
|
||
src/ascend/gmem_ascend.c | 22 ++++------------
|
||
src/ascend/gmem_ascend.h | 35 -------------------------
|
||
src/init.c | 56 ++++++++++++++++++++++++++++++++++++++++
|
||
src/libgmem.c | 18 +++++++------
|
||
7 files changed, 97 insertions(+), 77 deletions(-)
|
||
delete mode 100644 src/ascend/gmem_ascend.h
|
||
create mode 100644 src/init.c
|
||
|
||
diff --git a/Makefile.am b/Makefile.am
|
||
index e18c2fc..06b1435 100644
|
||
--- a/Makefile.am
|
||
+++ b/Makefile.am
|
||
@@ -23,7 +23,8 @@ include_HEADERS = include/libgmem.h
|
||
lib_LTLIBRARIES = libgmem.la
|
||
|
||
libgmem_la_SOURCES = \
|
||
- src/libgmem.c
|
||
+ src/libgmem.c \
|
||
+ src/init.c
|
||
|
||
libgmem_la_CFLAGS = $(AM_CFLAGS) \
|
||
-I$(top_srcdir)/include
|
||
diff --git a/include/gmem_common.h b/include/gmem_common.h
|
||
index 994f953..a400b37 100644
|
||
--- a/include/gmem_common.h
|
||
+++ b/include/gmem_common.h
|
||
@@ -13,22 +13,22 @@
|
||
#ifndef _GMEM_WAPPER_H_
|
||
#define _GMEM_WAPPER_H_
|
||
|
||
+#include <libgmem.h>
|
||
#include <stdlib.h>
|
||
|
||
-struct gm_msg {
|
||
- int behavior;
|
||
- unsigned long addr;
|
||
- size_t size;
|
||
- int hnid;
|
||
-};
|
||
-typedef struct gm_msg *gm_msg_t;
|
||
-
|
||
typedef struct {
|
||
int (*FreeEager)(void *userData, void *stream);
|
||
int (*Prefetch)(void *userData, void *stream);
|
||
int (*GetNumaId)(void);
|
||
} gmem_semantics;
|
||
extern gmem_semantics gmemSemantics;
|
||
+extern int gmem_fd;
|
||
+
|
||
+typedef struct hmadvise_arg gm_msg;
|
||
+typedef gm_msg *gm_msg_t;
|
||
+
|
||
+void init_device(void);
|
||
+void destory_device(void);
|
||
|
||
int gmemAdvise(void *userData);
|
||
#endif
|
||
diff --git a/include/libgmem.h b/include/libgmem.h
|
||
index ea313a3..4cd918b 100644
|
||
--- a/include/libgmem.h
|
||
+++ b/include/libgmem.h
|
||
@@ -20,14 +20,22 @@ __BEGIN_DECLS
|
||
/*
|
||
* TODO: Remove these "ifndef" after kernel upgrade
|
||
*/
|
||
-/*
|
||
- * __NR_madvise 441
|
||
- */
|
||
-#ifndef SYS_hmdvise
|
||
-# ifndef __NR_hmadvise
|
||
-# define __NR_hmadvise 441
|
||
-# endif
|
||
-# define SYS_hmadvise __NR_hmadvise
|
||
+#include <sys/ioctl.h>
|
||
+struct hmadvise_arg {
|
||
+ int hnid;
|
||
+ unsigned long start;
|
||
+ size_t len_in;
|
||
+ int behavior;
|
||
+};
|
||
+struct gmem_hnid_arg {
|
||
+ int *hnuma_id;
|
||
+};
|
||
+#ifndef GMEM_GET_HNUMA_ID
|
||
+ #define GMEM_GET_HNUMA_ID _IOW(0x55, 1, struct gmem_hnid_arg)
|
||
+#endif
|
||
+
|
||
+#ifndef GMEM_MADVISE
|
||
+ #define GMEM_MADVISE _IOW(0x55, 2, struct hmadvise_arg)
|
||
#endif
|
||
|
||
#ifndef MADV_PREFETCH
|
||
diff --git a/src/ascend/gmem_ascend.c b/src/ascend/gmem_ascend.c
|
||
index be54813..6181c07 100644
|
||
--- a/src/ascend/gmem_ascend.c
|
||
+++ b/src/ascend/gmem_ascend.c
|
||
@@ -19,13 +19,8 @@
|
||
|
||
#include <gmem_common.h>
|
||
|
||
-#include "gmem_ascend.h"
|
||
-
|
||
#define LIB_ASCENDCL_PATH "/usr/local/Ascend/ascend-toolkit/latest/aarch64-linux/lib64/libascendcl.so"
|
||
|
||
-void init_gmemSemantics(void) __attribute__ ((constructor));
|
||
-void exit_gmemSemantics(void) __attribute__ ((destructor));
|
||
-
|
||
typedef void (*Callback_fn)(void *userData);
|
||
typedef int (*ACLRT_LAUNCH_CB_FUNC)(Callback_fn fn, void *userData, int vlockType, void *stream);
|
||
ACLRT_LAUNCH_CB_FUNC aclrtLaunchCallback_fn;
|
||
@@ -43,26 +38,19 @@ int ascend_prefech(void *userData, void *stream)
|
||
|
||
int ascend_numaid(void)
|
||
{
|
||
- int ret, id, fd;
|
||
- struct rpg_hnid_arg arg;
|
||
-
|
||
- fd = open("/dev/remote_pager", O_RDWR);
|
||
- if (fd == -1) {
|
||
- perror("Get davinci_manager fd failed");
|
||
- return -ENXIO;
|
||
- }
|
||
+ int ret, id;
|
||
+ struct gmem_hnid_arg arg;
|
||
|
||
arg.hnuma_id = &id;
|
||
- ret = ioctl(fd, RPG_GET_HNUMA_ID, &arg);
|
||
+ ret = ioctl(gmem_fd, GMEM_GET_HNUMA_ID, &arg);
|
||
if (ret < 0) {
|
||
perror("Get hnuma id ioctl failed.");
|
||
return ret;
|
||
}
|
||
- close(fd);
|
||
return id;
|
||
}
|
||
|
||
-void init_gmemSemantics()
|
||
+void init_device()
|
||
{
|
||
handle = dlopen(LIB_ASCENDCL_PATH, RTLD_LAZY);
|
||
if (!handle) {
|
||
@@ -76,7 +64,7 @@ void init_gmemSemantics()
|
||
gmemSemantics.GetNumaId = ascend_numaid;
|
||
}
|
||
|
||
-void exit_gmemSemantics()
|
||
+void destory_device()
|
||
{
|
||
dlclose(handle);
|
||
}
|
||
diff --git a/src/ascend/gmem_ascend.h b/src/ascend/gmem_ascend.h
|
||
deleted file mode 100644
|
||
index 5348c2e..0000000
|
||
--- a/src/ascend/gmem_ascend.h
|
||
+++ /dev/null
|
||
@@ -1,35 +0,0 @@
|
||
-/*
|
||
- * Copyright (c) 2023 yangyanchao6@huawei.com
|
||
- * libgmem is licensed under Mulan PSL v2.
|
||
- * You can use this software according to the terms and conditions of the Mulan PSL v2.
|
||
- * You may obtain a copy of Mulan PSL v2 at:
|
||
- * http://license.coscl.org.cn/MulanPSL2
|
||
- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
|
||
- * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
||
- * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
||
- * See the Mulan PSL v2 for more details.
|
||
- */
|
||
-
|
||
-#ifndef _GMEM_ASCEND_H_
|
||
-#define _GMEM_ASCEND_H_
|
||
-
|
||
-/*
|
||
- * These definitions should be placed in the kernel,
|
||
- * which is a temporary implementation here.
|
||
- */
|
||
-
|
||
-#define RPG_MAGIC 0x55
|
||
-
|
||
-enum _RPG_IOCTL_CMD {
|
||
- _RPG_GET_HNUMA_ID = 1,
|
||
- _RPG_IOC_MAX_NR
|
||
-};
|
||
-
|
||
-struct rpg_hnid_arg {
|
||
- int *hnuma_id;
|
||
-};
|
||
-
|
||
-#define RPG_GET_HNUMA_ID \
|
||
- _IOW(RPG_MAGIC, _RPG_GET_HNUMA_ID, struct rpg_hnid_arg)
|
||
-
|
||
-#endif
|
||
diff --git a/src/init.c b/src/init.c
|
||
new file mode 100644
|
||
index 0000000..08b2800
|
||
--- /dev/null
|
||
+++ b/src/init.c
|
||
@@ -0,0 +1,56 @@
|
||
+/*
|
||
+ * Copyright (c) 2023 yangyanchao6@huawei.com
|
||
+ * libgmem is licensed under Mulan PSL v2.
|
||
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
|
||
+ * You may obtain a copy of Mulan PSL v2 at:
|
||
+ * http://license.coscl.org.cn/MulanPSL2
|
||
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
|
||
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
||
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
||
+ * See the Mulan PSL v2 for more details.
|
||
+ */
|
||
+
|
||
+#include <errno.h>
|
||
+#include <linux/mman.h>
|
||
+#include <stdio.h>
|
||
+#include <stdlib.h>
|
||
+#include <fcntl.h>
|
||
+
|
||
+#include <libgmem.h>
|
||
+#include <gmem_common.h>
|
||
+
|
||
+int gmem_fd;
|
||
+
|
||
+int init_libgmem(void) __attribute__ ((constructor));
|
||
+void exit_libgmem(void) __attribute__ ((destructor));
|
||
+
|
||
+int get_gmem_fd()
|
||
+{
|
||
+ gmem_fd = open("/dev/gmem", O_RDWR);
|
||
+ if (gmem_fd == -1) {
|
||
+ perror("Get gmem fd failed");
|
||
+ return -ENXIO;
|
||
+ }
|
||
+ return 0;
|
||
+}
|
||
+
|
||
+void release_gmem_fd()
|
||
+{
|
||
+ close(gmem_fd);
|
||
+}
|
||
+
|
||
+int init_libgmem(void)
|
||
+{
|
||
+ if (get_gmem_fd()) {
|
||
+ return -ENXIO;
|
||
+ }
|
||
+ init_device();
|
||
+ return 0;
|
||
+}
|
||
+
|
||
+void exit_libgmem(void)
|
||
+{
|
||
+ release_gmem_fd();
|
||
+ destory_device();
|
||
+}
|
||
+
|
||
diff --git a/src/libgmem.c b/src/libgmem.c
|
||
index a1a1640..9a34de0 100644
|
||
--- a/src/libgmem.c
|
||
+++ b/src/libgmem.c
|
||
@@ -23,17 +23,19 @@ gmem_semantics gmemSemantics;
|
||
static void mix_userData(gm_msg_t userData, unsigned long addr, size_t length, int hnid, int behavior)
|
||
{
|
||
userData->behavior = behavior;
|
||
- userData->addr = addr;
|
||
- userData->size = length;
|
||
+ userData->start = addr;
|
||
+ userData->len_in = length;
|
||
userData->hnid = hnid;
|
||
}
|
||
|
||
int gmemAdvise(void *userData)
|
||
{
|
||
- gm_msg_t msg = (gm_msg_t)userData;
|
||
- int ret = syscall(SYS_hmadvise, msg->hnid, msg->addr, msg->size, msg->behavior);
|
||
+ int ret;
|
||
+ gm_msg msg = *(gm_msg_t)userData;
|
||
+
|
||
+ ret = ioctl(gmem_fd, GMEM_MADVISE, msg);
|
||
if(ret) {
|
||
- printf("hmadvise failed: addr:%lx, size:%lx, behavior:%d, hnid:%d\n", msg->addr, msg->size, msg->behavior, msg->hnid);
|
||
+ printf("hmadvise failed: addr:%lx, size:%lx, behavior:%d, hnid:%d\n", msg.start, msg.len_in, msg.behavior, msg.hnid);
|
||
}
|
||
return ret;
|
||
}
|
||
@@ -41,7 +43,7 @@ int gmemAdvise(void *userData)
|
||
int gmemFreeEager(unsigned long addr, size_t length, void *stream)
|
||
{
|
||
int ret = -1;
|
||
- gm_msg_t userData = (gm_msg_t)malloc(sizeof(struct gm_msg));
|
||
+ gm_msg_t userData = (gm_msg_t)malloc(sizeof(gm_msg));
|
||
mix_userData(userData, addr, length, -1, MADV_DONTNEED);
|
||
if (!stream) {
|
||
ret = gmemAdvise(userData);
|
||
@@ -55,12 +57,12 @@ int gmemFreeEager(unsigned long addr, size_t length, void *stream)
|
||
int gmemPrefetch(unsigned long addr, size_t length, void *stream)
|
||
{
|
||
int ret = -1;
|
||
- gm_msg_t userData = (gm_msg_t)malloc(sizeof(struct gm_msg));
|
||
+ gm_msg_t userData = (gm_msg_t)malloc(sizeof(gm_msg));
|
||
mix_userData(userData, addr, length, gmemGetNumaId(), MADV_PREFETCH);
|
||
if (!stream) {
|
||
ret = gmemAdvise(userData);
|
||
} else if(gmemSemantics.Prefetch != NULL) {
|
||
- ret = gmemSemantics.FreeEager(userData,stream);
|
||
+ ret = gmemSemantics.Prefetch(userData, stream);
|
||
}
|
||
free(userData);
|
||
return ret;
|
||
--
|
||
2.41.0.windows.3
|
||
|