gala-gopher/add-support-to-pyroscope.patch

From 8099d91f2584fe6ee12eaea2d95b8d09a25075cb Mon Sep 17 00:00:00 2001
From: wo_cow <niuqianqian@huawei.com>
Date: Fri, 16 Dec 2022 19:42:55 +0800
Subject: [PATCH] add support to pyroscope

---
 gala-gopher.spec                              |   4 +-
 .../ebpf.probe/src/stackprobe/Makefile        |   2 +-
 .../src/stackprobe/conf/stackprobe.conf       |   1 +
 .../src/stackprobe/conf/stackprobe_conf.h     |   1 +
 .../src/stackprobe/conf/stackprobe_config.c   |   6 +
 .../ebpf.probe/src/stackprobe/flame_graph.c   | 193 +++++++++++++++++-
 .../ebpf.probe/src/stackprobe/flame_graph.h   |   6 +-
 .../ebpf.probe/src/stackprobe/stackprobe.c    |  13 +-
 .../ebpf.probe/src/stackprobe/stackprobe.h    |   8 +
 9 files changed, 221 insertions(+), 13 deletions(-)

diff --git a/gala-gopher.spec b/gala-gopher.spec
index a63351d..b8ed348 100644
--- a/gala-gopher.spec
+++ b/gala-gopher.spec
@@ -14,8 +14,8 @@ BuildRequires: clang >= 10.0.1
 BuildRequires: llvm
 BuildRequires: libconfig-devel librdkafka-devel libmicrohttpd-devel
 BuildRequires: uthash-devel libbpf libbpf-devel log4cplus-devel
-BuildRequires: java-1.8.0-openjdk-devel
-Requires: bash glibc elfutils zlib elfutils-devel bpftool iproute erlang-eflame
+BuildRequires: java-1.8.0-openjdk-devel libcurl-devel
+Requires: bash glibc elfutils zlib elfutils-devel bpftool iproute erlang-eflame libcurl
 
 %description
 gala-gopher is a low-overhead eBPF-based probes framework
diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/Makefile b/src/probes/extends/ebpf.probe/src/stackprobe/Makefile
index a36224d..c5758fe 100644
--- a/src/probes/extends/ebpf.probe/src/stackprobe/Makefile
+++ b/src/probes/extends/ebpf.probe/src/stackprobe/Makefile
@@ -31,7 +31,7 @@ deps: $(DEPS)
 
 app: $(APP)
 %: %.c $(SRC_C)
-	$(CC) $(CFLAGS) $(patsubst %.cpp, %.o, $(SRC_CPLUS))  $(INCLUDES) $^ $(LDFLAGS) $(LINK_TARGET) -o $@
+	$(CC) $(CFLAGS) $(patsubst %.cpp, %.o, $(SRC_CPLUS))  $(INCLUDES) $^ $(LDFLAGS) $(LINK_TARGET) -lcurl -o $@
 	@echo $@ "compiling completed."
 clean:
 	rm -rf $(DEPS)
diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe.conf b/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe.conf
index a2edd5d..ab79572 100644
--- a/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe.conf
+++ b/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe.conf
@@ -5,6 +5,7 @@ general =
     svg_dir = "/var/log/gala-gopher/stacktrace";
     flame_dir = "/var/log/gala-gopher/flamegraph";
     debug_dir = "/usr/lib/debug";
+    pyroscope_server = "localhost:4040";
 };
 
 flame_name =
diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_conf.h b/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_conf.h
index fa39dec..6c1787b 100644
--- a/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_conf.h
+++ b/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_conf.h
@@ -35,6 +35,7 @@ typedef struct {
     char svgDir[PATH_LEN];
     char flameDir[PATH_LEN];
     char debugDir[PATH_LEN];
+    char pyroscopeServer[PATH_LEN];
 } GeneralConfig;
 
 typedef struct {
diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_config.c b/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_config.c
index c191187..a9b5dfb 100644
--- a/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_config.c
+++ b/src/probes/extends/ebpf.probe/src/stackprobe/conf/stackprobe_config.c
@@ -205,6 +205,12 @@ static int configLoadGeneral(void *config, config_setting_t *settings)
     }
     (void)strncpy(generalConfig->debugDir, strVal, PATH_LEN - 1);
 
+    ret = config_setting_lookup_string(settings, "pyroscope_server", &strVal);
+    if (ret == 0) {
+        strVal = ""; // will not post to pyroscope
+    }
+    (void)strncpy(generalConfig->pyroscopeServer, strVal, PATH_LEN - 1);
+
     return 0;
 }
 
diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.c b/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.c
index 870c6d9..126b98d 100644
--- a/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.c
+++ b/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.c
@@ -25,6 +25,7 @@
 #include <sys/wait.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <curl/curl.h>
 
 #ifdef BPF_PROG_KERN
 #undef BPF_PROG_KERN
@@ -37,6 +38,27 @@
 #include "bpf.h"
 #include "flame_graph.h"
 
+struct post_info_s {
+    int post_flag;
+    int sk;
+    int remain_size;
+    char *buf_start;
+    char *buf;
+    CURL *curl;
+};
+
+struct MemoryStruct {
+  char *memory;
+  size_t size;
+};
+
+static char *appname[STACK_SVG_MAX] = {
+    "gala-gopher-oncpu",
+    "gala-gopher-offcpu",
+    "gala-gopher-io",
+    "gala-gopher-memleak"
+};
+
 #if 1
 
 static char __test_flame_graph_flags(struct stack_svg_mng_s *svg_mng, u32 flags)
@@ -170,8 +192,10 @@ static void __reopen_flame_graph_file(struct stack_svg_mng_s *svg_mng)
 
 #define HISTO_TMP_LEN   (2 * STACK_SYMBS_LEN)
 static char __histo_tmp_str[HISTO_TMP_LEN];
+
+#define POST_MAX_LEN 2048
 static int __do_wr_stack_histo(struct stack_svg_mng_s *svg_mng,
-                               struct stack_trace_histo_s *stack_trace_histo, int first)
+                               struct stack_trace_histo_s *stack_trace_histo, int first, struct post_info_s *post_info)
 {
     FILE *fp = __get_flame_graph_fp(svg_mng);
     if (!fp) {
@@ -188,34 +212,166 @@ static int __do_wr_stack_histo(struct stack_svg_mng_s *svg_mng,
         (void)snprintf(__histo_tmp_str, HISTO_TMP_LEN, "\n%s %llu",
                 stack_trace_histo->stack_symbs_str, stack_trace_histo->count);
     }
+    if (post_info->post_flag) {
+        (void)__snprintf(&post_info->buf, post_info->remain_size, &post_info->remain_size, "%s", __histo_tmp_str);
+    }
+
     (void)fputs(__histo_tmp_str, fp);
     return 0;
 }
 
-static void __do_wr_flamegraph(struct stack_svg_mng_s *svg_mng, struct stack_trace_histo_s *head)
+static size_t __write_memory_cb(void *contents, size_t size, size_t nmemb, void *userp)
+{
+    size_t realsize = size * nmemb;
+    struct MemoryStruct *mem = (struct MemoryStruct *)userp;
+    
+    char *ptr = realloc(mem->memory, mem->size + realsize + 1);
+    if(!ptr) {
+        /* out of memory! */
+        printf("not enough memory (realloc returned NULL)\n");
+        return 0;
+    }
+    
+    mem->memory = ptr;
+    memcpy(&(mem->memory[mem->size]), contents, realsize);
+    mem->size += realsize;
+    mem->memory[mem->size] = 0;
+    
+    return realsize;
+}
+ 
+// http://localhost:4040/ingest?name=gala-gopher-oncpu&from=1671189474&until=1671189534
+static int __build_url(char *url, struct post_server_s *post_server, int en_type)
+{
+    time_t now, before;
+    (void)time(&now);
+    if (post_server->last_post_ts == 0) {
+        before = now - 60; // 60s
+    } else {
+        before = post_server->last_post_ts + 1;
+    }
+    post_server->last_post_ts = now;
+
+    (void)snprintf(url, LINE_BUF_LEN, 
+        "http://%s/ingest?name=%s&from=%ld&until=%ld",
+        post_server->host,
+        appname[en_type],
+        (long)before,
+        (long)now);
+    return 0;
+}
+
+
+static void __curl_post(struct post_server_s *post_server, struct post_info_s *post_info, int en_type)
+{
+    CURLcode res;
+    CURL *curl = post_info->curl;
+    if (curl == NULL) {
+        return;
+    }
+    long post_len = (long)strlen(post_info->buf_start);
+    if (post_len == 0) {
+        DEBUG("[FLAMEGRAPH]: buf is null. No need to curl post post to %s\n", appname[en_type]);
+        return;
+    }
+
+    char url[LINE_BUF_LEN] = {0};
+    __build_url(url, post_server, en_type);
+    struct MemoryStruct chunk;
+    chunk.memory = malloc(1);  /* will be grown as needed by realloc above */
+    chunk.size = 0;    /* no data at this point */
+
+    //curl_easy_setopt(curl, CURLOPT_URL, post_server->host);
+    curl_easy_setopt(curl, CURLOPT_URL, url);
+
+    curl_easy_setopt(curl, CURLOPT_TIMEOUT, post_server->timeout);
+
+    /* send all data to this function */
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, __write_memory_cb);
+
+    /* we pass our 'chunk' struct to the callback function */
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&chunk);
+
+    /* some servers do not like requests that are made without a user-agent
+    field, so we provide one */
+    curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0");
+
+    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_info->buf_start);
+
+    /* if we do not provide POSTFIELDSIZE, libcurl will strlen() by
+    itself */
+    curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, post_len);
+
+    /* Perform the request, res will get the return code */
+    res = curl_easy_perform(curl);
+    /* Check for errors */
+    if(res != CURLE_OK) {
+        ERROR("[FLAMEGRAPH]: curl post failed: %s\n", curl_easy_strerror(res));
+    } else {
+        INFO("[FLAMEGRAPH]: curl post post to %s success\n", appname[en_type]);
+    }
+
+    /* always cleanup */
+    curl_easy_cleanup(curl);
+    if (chunk.memory) {
+        free(chunk.memory);
+        chunk.memory = NULL;
+    }
+    free(post_info->buf_start);
+    post_info->buf_start = NULL;
+
+    return;
+}
+
+static void __init_curl_handle(struct post_server_s *post_server, struct post_info_s *post_info)
+{
+    if (post_server == NULL || post_server->post_flag == 0) {
+        return;
+    }
+
+    post_info->curl = curl_easy_init();
+    if(post_info->curl) {
+        post_info->buf = (char *)malloc(POST_MAX_LEN);
+        post_info->buf_start = post_info->buf;
+        if (post_info->buf != NULL) {
+            post_info->buf[0] = 0;
+            post_info->post_flag = 1;
+        }
+    }
+}
+
+static void __do_wr_flamegraph(struct stack_svg_mng_s *svg_mng, struct stack_trace_histo_s *head,
+    struct post_server_s *post_server, int en_type)
 {
     int first_flag = 0;
+    struct post_info_s post_info = {.remain_size = POST_MAX_LEN, .post_flag = 0};
 
     if (__test_flame_graph_flags(svg_mng, FLAME_GRAPH_NEW)) {
         first_flag = 1;
     }
 
-    struct stack_trace_histo_s *item, *tmp;
+    __init_curl_handle(post_server, &post_info);
 
+    struct stack_trace_histo_s *item, *tmp;
     H_ITER(head, item, tmp) {
-        (void)__do_wr_stack_histo(svg_mng, item, first_flag);
+        (void)__do_wr_stack_histo(svg_mng, item, first_flag, &post_info);
         first_flag = 0;
     }
-
+    if (post_info.post_flag) {
+        __curl_post(post_server, &post_info, en_type);
+    }
+    
     __flush_flame_graph_file(svg_mng);
     __reset_flame_graph_flags(svg_mng, ~FLAME_GRAPH_NEW);
 }
 
 #endif
 
-void wr_flamegraph(struct stack_svg_mng_s *svg_mng, struct stack_trace_histo_s *head, int en_type)
+void wr_flamegraph(struct stack_svg_mng_s *svg_mng, struct stack_trace_histo_s *head, int en_type,
+    struct post_server_s *post_server)
 {
-    __do_wr_flamegraph(svg_mng, head);
+    __do_wr_flamegraph(svg_mng, head, post_server, en_type);
+
     if (is_svg_tmout(svg_mng)) {
         (void)create_svg_file(svg_mng,
                               __get_flame_graph_file(svg_mng), en_type);
@@ -251,3 +407,26 @@ int set_flame_graph_path(struct stack_svg_mng_s *svg_mng, const char* path, cons
     return 0;
 }
 
+int set_post_server(struct post_server_s *post_server, const char *server_str)
+{
+    if (server_str == NULL) {
+        return -1;
+    }
+
+    char *p = strrchr(server_str, ':');
+    if (p == NULL) {
+        return -1;
+    }
+
+    curl_global_init(CURL_GLOBAL_ALL);
+    post_server->post_flag = 1;
+    post_server->timeout = 3;
+    (void)strcpy(post_server->host, server_str);
+
+    return 0;
+}
+
+void clean_post_server()
+{
+    curl_global_cleanup();
+}
diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.h b/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.h
index a9ed999..ea29107 100644
--- a/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.h
+++ b/src/probes/extends/ebpf.probe/src/stackprobe/flame_graph.h
@@ -20,7 +20,9 @@
 #include "svg.h"
 #include "stackprobe.h"
 
-void wr_flamegraph(struct stack_svg_mng_s *svg_mng, struct stack_trace_histo_s *head, int en_type);
+void wr_flamegraph(struct stack_svg_mng_s *svg_mng, struct stack_trace_histo_s *head, int en_type,
+    struct post_server_s *post_server);
 int set_flame_graph_path(struct stack_svg_mng_s *svg_mng, const char* path, const char *flame_name);
-
+int set_post_server(struct post_server_s *post_server, const char *pyroscopeServer);
+void clean_post_server();
 #endif
diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.c b/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.c
index fa37a72..a4733e6 100644
--- a/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.c
+++ b/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.c
@@ -753,6 +753,10 @@ static void destroy_stack_trace(struct stack_trace_s **ptr_st)
         return;
     }
 
+    if (st->post_server.post_flag) {
+        clean_post_server();
+    }
+
     for (int cpu = 0; cpu < st->cpus_num; cpu++) {
         if (st->pmu_fd[cpu] > 0) {
             ioctl(st->pmu_fd[cpu], PERF_EVENT_IOC_DISABLE);
@@ -843,6 +847,13 @@ static struct stack_trace_s *create_stack_trace(StackprobeConfig *conf)
     }
 #endif
 
+    if (set_post_server(&st->post_server, conf->generalConfig->pyroscopeServer) != 0) {
+        INFO("[STACKPROBE]: Do not post to Pyroscope Server.\n");
+        st->post_server.post_flag = 0;
+    } else {
+        INFO("[STACKPROBE]: Will post to Pyroscope Server: %s.\n", conf->generalConfig->pyroscopeServer);
+    }
+
     st->elf_reader = create_elf_reader(conf->generalConfig->debugDir);
     if (!st->elf_reader) {
         goto err;
@@ -1317,7 +1328,7 @@ static void switch_stackmap()
             continue;
         }
         (void)stack_id2histogram(st, i, st->is_stackmap_a);
-        wr_flamegraph(st->svg_stack_traces[i]->svg_mng, st->svg_stack_traces[i]->histo_tbl, i);
+        wr_flamegraph(st->svg_stack_traces[i]->svg_mng, st->svg_stack_traces[i]->histo_tbl, i, &st->post_server);
         clear_raw_stack_trace(st->svg_stack_traces[i], st->is_stackmap_a);
     }
     record_running_ctx(st);
diff --git a/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.h b/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.h
index 1f85225..657b6e7 100644
--- a/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.h
+++ b/src/probes/extends/ebpf.probe/src/stackprobe/stackprobe.h
@@ -102,6 +102,13 @@ struct svg_stack_trace_s {
     struct stack_trace_histo_s *histo_tbl;
 };
 
+struct post_server_s {
+    char post_flag;
+    long timeout; // sec
+    char host[PATH_LEN];
+    time_t last_post_ts;
+};
+
 struct stack_trace_s {
     char pad[3];
     int cpus_num;
@@ -112,6 +119,7 @@ struct stack_trace_s {
     int stackmap_b_fd;
     u64 convert_stack_count;
     time_t running_times;
+    struct post_server_s post_server;
 
     struct svg_stack_trace_s *svg_stack_traces[STACK_SVG_MAX];
     struct ksymb_tbl_s *ksymbs;
-- 
2.33.0