From 388da3f3fc05357f1fbfc8a12ce40ad198397332 Mon Sep 17 00:00:00 2001 From: h00465007 Date: Wed, 24 Apr 2024 10:45:15 +0800 Subject: [PATCH] refactor(cadvisor): remove the dependency on the python-libconf --- gala-gopher.spec | 2 +- .../cadvisor.probe/cadvisor_probe.conf | 449 +++++++++--------- .../cadvisor.probe/cadvisor_probe.py | 26 +- 3 files changed, 240 insertions(+), 237 deletions(-) diff --git a/gala-gopher.spec b/gala-gopher.spec index 2c11562..24f6bae 100644 --- a/gala-gopher.spec +++ b/gala-gopher.spec @@ -81,7 +81,7 @@ Requires: libcurl Requires: python3-psycopg2 python3-yaml net-tools %endif %if !0%{?without_container} -Requires: cadvisor python3-libconf python3-requests net-tools util-linux +Requires: cadvisor python3-requests net-tools util-linux %endif %if !0%{?without_postgre_sli} Requires: iproute diff --git a/src/probes/extends/python.probe/cadvisor.probe/cadvisor_probe.conf b/src/probes/extends/python.probe/cadvisor.probe/cadvisor_probe.conf index 1ff2283..79686ee 100644 --- a/src/probes/extends/python.probe/cadvisor.probe/cadvisor_probe.conf +++ b/src/probes/extends/python.probe/cadvisor.probe/cadvisor_probe.conf @@ -1,224 +1,225 @@ -version = "1.0.0" -measurements: -( - { - table_name: "container_cpu", - entity_name: "container", - fields: - ( - { - description: "container id", - type: "key", - name: "container_id", - }, - { - description: "cpu", - type: "label", - name: "cpu", - }, - { - description: "Cumulative system cpu time consumed in seconds", - type: "counter", - name: "cpu_system_seconds_total", - }, - { - description: "Cumulative cpu time consumed in seconds", - type: "counter", - name: "cpu_usage_seconds_total", - }, - { - description: "Cumulative user cpu time consumed in seconds", - type: "counter", - name: "cpu_user_seconds_total", - }, - { - description: "Total time duration the container has been throttled", - type: "counter", - name: "cpu_cfs_throttled_seconds_total" - } - ) - }, - { - table_name: "container_fs", - entity_name: "container", - fields: - ( - { - description: "container id", - type: "key", - name: "container_id", - }, - { - description: "device", - type: "label", - name: "device", - }, - { - description: "Number of available Inodes", - type: "gauge", - name: "fs_inodes_free", - }, - { - description: "Number of Inodes", - type: "gauge", - name: "fs_inodes_total", - }, - { - description: "Cumulative count of seconds spent reading", - type: "counter", - name: "fs_read_seconds_total", - }, - { - description: "Cumulative count of seconds spent writing", - type: "counter", - name: "fs_write_seconds_total", - }, - { - description: "Cumulative count of bytes read", - type: "counter", - name: "fs_reads_bytes_total", - }, - { - description: "Cumulative count of bytes written", - type: "counter", - name: "fs_writes_bytes_total", - } - ) - }, - { - table_name: "container_memory", - entity_name: "container", - fields: - ( - { - description: "container id", - type: "key", - name: "container_id", - }, - { - description: "Number of bytes of page cache memory", - type: "gauge", - name: "memory_cache", - }, - { - description: "Size of memory mapped files in bytes", - type: "gauge", - name: "memory_mapped_file", - }, - { - description: "Size of RSS in byte", - type: "gauge", - name: "memory_rss", - }, - { - description: "Current memory usage in bytes, including all memory regardless of when it was accessed", - type: "gauge", - name: "memory_usage_bytes", - }, - { - description: "Current working set in bytes", - type: "gauge", - name: "memory_working_set_bytes", - } - ) - }, - { - table_name: "container_network", - entity_name: "container", - fields: - ( - { - description: "container id", - type: "key", - name: "container_id", - }, - { - description: "interface", - type: "label", - name: "interface", - }, - { - description: "Cumulative count of bytes received", - type: "counter", - name: "network_receive_bytes_total", - }, - { - description: "Cumulative count of errors encountered while receiving", - type: "counter", - name: "network_receive_errors_total", - }, - { - description: "Cumulative count of packets dropped while receiving", - type: "counter", - name: "network_receive_packets_dropped_total", - }, - { - description: "Cumulative count of bytes transmitted", - type: "counter", - name: "network_transmit_bytes_total", - }, - { - description: "Cumulative count of errors encountered while transmitting", - type: "counter", - name: "network_transmit_errors_total", - }, - { - description: "Cumulative count of packets dropped while transmitting", - type: "counter", - name: "network_transmit_packets_dropped_total", - } - ) - }, - { - table_name: "container_oom", - entity_name: "container", - fields: - ( - { - description: "container id", - type: "key", - name: "container_id", - }, - { - description: "Count of out of memory events observed for the container", - type: "counter", - name: "oom_events_total", - } - ) - }, - { - table_name: "container_start", - entity_name: "container", - fields: - ( - { - description: "container id", - type: "key", - name: "container_id", - }, - { - description: "Start time of the container since unix epoch in seconds", - type: "gauge", - name: "start_time_seconds", - } - ) - }, - { - table_name: "container_file", - entity_name: "container", - fields: - ( - { - description: "container id", - type: "key", - name: "container_id", - }, - { - description: "Number of open file descriptors for the container", - type: "gauge", - name: "file_descriptors", - } - ) - } -) - +{ + "version": "1.0.0", + "measurements": + [ + { + "table_name": "container_cpu", + "entity_name": "container", + "fields": + [ + { + "description": "container id", + "type": "key", + "name": "container_id" + }, + { + "description": "cpu", + "type": "label", + "name": "cpu" + }, + { + "description": "Cumulative system cpu time consumed in seconds", + "type": "counter", + "name": "cpu_system_seconds_total" + }, + { + "description": "Cumulative cpu time consumed in seconds", + "type": "counter", + "name": "cpu_usage_seconds_total" + }, + { + "description": "Cumulative user cpu time consumed in seconds", + "type": "counter", + "name": "cpu_user_seconds_total" + }, + { + "description": "Total time duration the container has been throttled", + "type": "counter", + "name": "cpu_cfs_throttled_seconds_total" + } + ] + }, + { + "table_name": "container_fs", + "entity_name": "container", + "fields": + [ + { + "description": "container id", + "type": "key", + "name": "container_id" + }, + { + "description": "device", + "type": "label", + "name": "device" + }, + { + "description": "Number of available Inodes", + "type": "gauge", + "name": "fs_inodes_free" + }, + { + "description": "Number of Inodes", + "type": "gauge", + "name": "fs_inodes_total" + }, + { + "description": "Cumulative count of seconds spent reading", + "type": "counter", + "name": "fs_read_seconds_total" + }, + { + "description": "Cumulative count of seconds spent writing", + "type": "counter", + "name": "fs_write_seconds_total" + }, + { + "description": "Cumulative count of bytes read", + "type": "counter", + "name": "fs_reads_bytes_total" + }, + { + "description": "Cumulative count of bytes written", + "type": "counter", + "name": "fs_writes_bytes_total" + } + ] + }, + { + "table_name": "container_memory", + "entity_name": "container", + "fields": + [ + { + "description": "container id", + "type": "key", + "name": "container_id" + }, + { + "description": "Number of bytes of page cache memory", + "type": "gauge", + "name": "memory_cache" + }, + { + "description": "Size of memory mapped files in bytes", + "type": "gauge", + "name": "memory_mapped_file" + }, + { + "description": "Size of RSS in byte", + "type": "gauge", + "name": "memory_rss" + }, + { + "description": "Current memory usage in bytes, including all memory regardless of when it was accessed", + "type": "gauge", + "name": "memory_usage_bytes" + }, + { + "description": "Current working set in bytes", + "type": "gauge", + "name": "memory_working_set_bytes" + } + ] + }, + { + "table_name": "container_network", + "entity_name": "container", + "fields": + [ + { + "description": "container id", + "type": "key", + "name": "container_id" + }, + { + "description": "interface", + "type": "label", + "name": "interface" + }, + { + "description": "Cumulative count of bytes received", + "type": "counter", + "name": "network_receive_bytes_total" + }, + { + "description": "Cumulative count of errors encountered while receiving", + "type": "counter", + "name": "network_receive_errors_total" + }, + { + "description": "Cumulative count of packets dropped while receiving", + "type": "counter", + "name": "network_receive_packets_dropped_total" + }, + { + "description": "Cumulative count of bytes transmitted", + "type": "counter", + "name": "network_transmit_bytes_total" + }, + { + "description": "Cumulative count of errors encountered while transmitting", + "type": "counter", + "name": "network_transmit_errors_total" + }, + { + "description": "Cumulative count of packets dropped while transmitting", + "type": "counter", + "name": "network_transmit_packets_dropped_total" + } + ] + }, + { + "table_name": "container_oom", + "entity_name": "container", + "fields": + [ + { + "description": "container id", + "type": "key", + "name": "container_id" + }, + { + "description": "Count of out of memory events observed for the container", + "type": "counter", + "name": "oom_events_total" + } + ] + }, + { + "table_name": "container_start", + "entity_name": "container", + "fields": + [ + { + "description": "container id", + "type": "key", + "name": "container_id" + }, + { + "description": "Start time of the container since unix epoch in seconds", + "type": "gauge", + "name": "start_time_seconds" + } + ] + }, + { + "table_name": "container_file", + "entity_name": "container", + "fields": + [ + { + "description": "container id", + "type": "key", + "name": "container_id" + }, + { + "description": "Number of open file descriptors for the container", + "type": "gauge", + "name": "file_descriptors" + } + ] + } + ] +} diff --git a/src/probes/extends/python.probe/cadvisor.probe/cadvisor_probe.py b/src/probes/extends/python.probe/cadvisor.probe/cadvisor_probe.py index 5adbaf2..c4dcb45 100755 --- a/src/probes/extends/python.probe/cadvisor.probe/cadvisor_probe.py +++ b/src/probes/extends/python.probe/cadvisor.probe/cadvisor_probe.py @@ -7,7 +7,8 @@ import subprocess import os import io import requests -import libconf +import json +import re import ipc CONTAINER_ID_LEN = 64 @@ -49,13 +50,13 @@ def convert_meta(): global g_meta meta_path = os.path.join("/etc/gala-gopher/extend_probes/cadvisor_probe.conf") with io.open(meta_path, encoding='utf-8') as f: - meta = libconf.load(f) + meta = json.load(f) g_meta = dict() - for measure in meta.measurements: - g_meta[measure.table_name] = dict() - for field in measure.fields: + for measure in meta.get("measurements"): + g_meta[measure.get("table_name")] = dict() + for field in measure.get("fields"): try: - g_meta[measure.table_name][field.name] = field.type + g_meta[measure.get("table_name")][field.get("name")] = field.get("type") except KeyError: # main will catch the exception raise @@ -164,20 +165,21 @@ class CadvisorProbe(): if table_name not in g_metric: g_metric[table_name] = dict() - metric_str = libconf.loads(line[(line.index("{") + 1):line.index("} ")]) + metric_str = line[line.index("{"):line.index("} ")+1] + metric_dict = json.loads(re.sub(r'(\w+)=', r'"\1":', metric_str)) # cadvisor metric id is cgroup path of container - if metric_str.id not in self.cgroup_path_map.keys(): - continue; + if metric_dict.get("id") not in self.cgroup_path_map.keys(): + continue label_key= '' for field_name, field_type in g_meta[table_name].items(): - if field_type == LABEL and field_name in metric_str: - label_key += "_" + metric_str[field_name] + if field_type == LABEL and field_name in metric_dict: + label_key += "_" + metric_dict[field_name] if label_key == '': label_key = LABEL - container_id = self.cgroup_path_map[metric_str.id] + container_id = self.cgroup_path_map[metric_dict.get("id")] if container_id not in g_metric[table_name]: g_metric[table_name][container_id] = dict() -- 2.28.0.windows.1