From 8e84d5ba4aad02509165cb1926091baa3630e418 Mon Sep 17 00:00:00 2001
From: hejingxian <hejingxian@huawei.com>
Date: Fri, 14 Feb 2020 16:42:01 +0800
Subject: [PATCH] feature: add ability for unblock socket communication

This will be used with user interface and also can be used as API for users
to create their own scripts on top of. The socket communication can be used
for receiving data about IRQs-to-CPUs assignments and setup, as well as
setting some options during runtime.

Socket address: /var/run/uvp_irqbalance.socket

Data to send to socket:
    settings sleep <int>: set new sleep interval value
    settings cpus <cpumask> <cpumask> ... : ban listed CPUs from
        IRQ handling (old values are forgotten, not added to)
    settings ban irqs <irq1> <irq2> ... : ban listed IRQs from balancing (old
        values are forgotten, not added to)

Signed-off-by: Xu Yandong <xuyandong2@huawei.com>
Signed-off-by: He Jingxian <hejingxian@huawei.com>
---
 Makefile.am  |   4 +-
 classify.c   |   2 +-
 cputree.c    |  20 ++--
 irqbalance.c |  33 +++++--
 irqbalance.h |   1 +
 sockapi.c    | 297 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 sockapi.h    |  19 ++++
 7 files changed, 359 insertions(+), 17 deletions(-)
 create mode 100644 sockapi.c
 create mode 100644 sockapi.h

diff --git a/Makefile.am b/Makefile.am
index 46e7173..c7be22d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -30,7 +30,7 @@ UI_DIR = ui
 AM_CFLAGS = $(LIBCAP_NG_CFLAGS) $(GLIB2_CFLAGS)
 AM_CPPFLAGS = -I${top_srcdir} -W -Wall -Wshadow -Wformat -Wundef -D_GNU_SOURCE
 noinst_HEADERS = bitmap.h constants.h cpumask.h irqbalance.h non-atomic.h \
-	types.h $(UI_DIR)/helpers.h $(UI_DIR)/irqbalance-ui.h $(UI_DIR)/ui.h
+	types.h sockapi.h $(UI_DIR)/helpers.h $(UI_DIR)/irqbalance-ui.h $(UI_DIR)/ui.h
 sbin_PROGRAMS = irqbalance
 
 if IRQBALANCEUI
@@ -38,7 +38,7 @@ sbin_PROGRAMS += irqbalance-ui
 endif
 
 irqbalance_SOURCES = activate.c bitmap.c classify.c cputree.c irqbalance.c \
-	irqlist.c numa.c placement.c procinterrupts.c rules_config.c hint_verify.c
+	irqlist.c numa.c placement.c procinterrupts.c rules_config.c hint_verify.c sockapi.c
 irqbalance_LDADD = $(LIBCAP_NG_LIBS) $(GLIB2_LIBS)
 if IRQBALANCEUI
 irqbalance_ui_SOURCES = $(UI_DIR)/helpers.c $(UI_DIR)/irqbalance-ui.c \
diff --git a/classify.c b/classify.c
index 91ff022..b0bb3b4 100644
--- a/classify.c
+++ b/classify.c
@@ -37,7 +37,7 @@ static GList *interrupts_db = NULL;
 static GList *banned_irqs = NULL;
 GList *cl_banned_irqs = NULL;
 static GList *cl_banned_modules = NULL;
-static GList *vm_banned_irqs = NULL;
+GList *vm_banned_irqs = NULL;
 extern int need_add_single;
 
 #define SYSFS_DIR "/sys"
diff --git a/cputree.c b/cputree.c
index 741c7c8..5a96e30 100644
--- a/cputree.c
+++ b/cputree.c
@@ -40,6 +40,8 @@
 
 extern char *banned_cpumask_from_ui;
 extern char *cpu_ban_string;
+extern int use_unblock_socket;
+extern int is_set_banned_cpumask_from_ui;
 
 GList *cpus;
 GList *cache_domains;
@@ -78,12 +80,18 @@ static void setup_banned_cpus(void)
 	cpus_clear(nohz_full);
 
 	/* A manually specified cpumask overrides auto-detection. */
-	if (cpu_ban_string != NULL && banned_cpumask_from_ui != NULL) {
-		cpulist_parse(banned_cpumask_from_ui,
-			strlen(banned_cpumask_from_ui), banned_cpus);
-		goto out;
-	}
-	if (getenv("IRQBALANCE_BANNED_CPUS"))  {
+		if (!use_unblock_socket && cpu_ban_string != NULL && banned_cpumask_from_ui != NULL) {
+			cpulist_parse(banned_cpumask_from_ui,
+			strlen(banned_cpumask_from_ui), banned_cpus);
+			goto out;
+		}
+		if (use_unblock_socket && is_set_banned_cpumask_from_ui) {
+			cpumask_parse_user(banned_cpumask_from_ui,
+			strlen(banned_cpumask_from_ui), banned_cpus);
+			goto out;
+	}
+
+	if (getenv("IRQBALANCE_BANNED_CPUS")) {
 		cpumask_parse_user(getenv("IRQBALANCE_BANNED_CPUS"), strlen(getenv("IRQBALANCE_BANNED_CPUS")), banned_cpus);
 		goto out;
 	}
diff --git a/irqbalance.c b/irqbalance.c
index dc8307d..1774eda 100644
--- a/irqbalance.c
+++ b/irqbalance.c
@@ -42,6 +42,7 @@
 #include <cap-ng.h>
 #endif
 #include "irqbalance.h"
+#include "sockapi.h"
 
 volatile int keep_going = 1;
 volatile int ban_pci_assigned_irq = 1;
@@ -73,6 +74,7 @@ GMainLoop *main_loop;
 
 char *cpu_ban_string = NULL;
 char *banned_cpumask_from_ui = NULL;
+int use_unblock_socket = 1;
 
 static void sleep_approx(int seconds)
 {
@@ -109,6 +111,7 @@ struct option lopts[] = {
 	{"notclearhint", 0, NULL, 'n'},
 	{"migrateval", 1, NULL, 'e'},
 	{"loadlimit", 1, NULL, 'g'},
+	{"blocksocket", 0, NULL, 'k'},
 	{0, 0, 0, 0}
 };
 
@@ -116,8 +119,8 @@ static void usage(void)
 {
 	log(TO_CONSOLE, LOG_INFO, "irqbalance [--oneshot | -o] [--debug | -d] [--foreground | -f] [--journal | -j] [--hintpolicy | -h <subset>]\n");
 	log(TO_CONSOLE, LOG_INFO, "	[--powerthresh= | -p <off> | <n>] [--banirq= | -i <n>] [--banmod= | -m <module>] [--policyscript= | -l <script>]\n");
-	log(TO_CONSOLE, LOG_INFO, "     [--pid= | -s <file>] [--deepestcache= | -c <n>] [--interval= | -t <n>] [--verifyhint= | -v n]\n");
-	log(TO_CONSOLE, LOG_INFO, "     [--rulesconfig= | -r <config>] [--notclearhint | -n] [--migrateval= | -e <n>] [--loadlimit= | -g <n>]\n");
+	log(TO_CONSOLE, LOG_INFO, " [--pid= | -s <file>] [--deepestcache= | -c <n>] [--interval= | -t <n>] [--verifyhint= | -v n] [--blocksocket | -k]\n");
+	log(TO_CONSOLE, LOG_INFO, " [--rulesconfig= | -r <config>] [--notclearhint | -n] [--migrateval= | -e <n>] [--loadlimit= | -g <n>]\n");
 }
 
 static void version(void)
@@ -132,7 +135,7 @@ static void parse_command_line(int argc, char **argv)
 	unsigned long val;
 
 	while ((opt = getopt_long(argc, argv,
-		"odfjVni:p:s:c:b:l:m:t:h:v:r:e:g:",
+		"odfjVni:p:s:c:b:l:m:t:h:v:r:e:g:k",
 		lopts, &longind)) != -1) {
 
 		switch(opt) {
@@ -240,7 +243,9 @@ static void parse_command_line(int argc, char **argv)
 			case 'g':
 				load_limit = strtoul(optarg, NULL, 10);
 				break;
-
+			case 'k':
+				use_unblock_socket = 0;
+				break;
 		}
 	}
 }
@@ -562,9 +567,12 @@ gboolean sock_handle(gint fd, GIOCondition condition, gpointer user_data __attri
 			if (!setup)
 				goto out_close;
 			snprintf(setup, strlen("SLEEP  ") + 11 + 1, "SLEEP %d ", sleep_interval);
-			if(g_list_length(cl_banned_irqs) > 0) {
+			if (g_list_length(cl_banned_irqs) > 0) {
 				for_each_irq(cl_banned_irqs, get_irq_data, &setup);
 			}
+			if (g_list_length(vm_banned_irqs) > 0) {
+				for_each_irq(vm_banned_irqs, get_irq_data, &setup);
+			}
 			cpumask_scnprintf(banned, 512, banned_cpus);
 			newptr = realloc(setup, strlen(setup) + strlen(banned) + 7 + 1);
 			if (!newptr)
@@ -827,10 +835,18 @@ int main(int argc, char** argv)
 	parse_proc_interrupts();
 	parse_proc_stat();
 
-	if (init_socket()) {
-		ret = EXIT_FAILURE;
-		goto out;
+	if (use_unblock_socket) {
+		if (init_unblock_socket()) {
+			ret = EXIT_FAILURE;
+			goto out;
+		}
+	} else {
+		if (init_socket()) {
+			ret = EXIT_FAILURE;
+			goto out;
+		}
 	}
+
 	log(TO_ALL, LOG_INFO, "irqbalance start scan.\n");
 	update_interval_and_count();
 	main_loop = g_main_loop_new(NULL, FALSE);
@@ -852,6 +868,7 @@ out:
 		close(socket_fd);
 	if (socket_name[0])
 		unlink(socket_name);
+	free(banned_cpumask_from_ui);
 
 	return ret;
 }
diff --git a/irqbalance.h b/irqbalance.h
index fc42a9b..7473ee7 100644
--- a/irqbalance.h
+++ b/irqbalance.h
@@ -65,6 +65,7 @@ extern GList *cache_domains;
 extern GList *cpus;
 extern int numa_avail;
 extern GList *cl_banned_irqs;
+extern GList *vm_banned_irqs;
 
 extern int debug_mode;
 extern int journal_logging;
diff --git a/sockapi.c b/sockapi.c
new file mode 100644
index 0000000..9891978
--- /dev/null
+++ b/sockapi.c
@@ -0,0 +1,297 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/epoll.h>
+#include <pthread.h>
+
+#include "sockapi.h"
+#include "irqbalance.h"
+
+volatile int is_set_banned_cpumask_from_ui = 0;
+
+extern char *banned_cpumask_from_ui;
+extern int socket_fd;
+extern char socket_name[64];
+extern int sleep_interval;
+extern GList *cl_banned_irqs;
+extern GList *vm_banned_irqs;
+extern cpumask_t banned_cpus;
+extern int keep_going;
+int cur_fd_count = 0;
+
+static void fill_banned_cpus(char *cpustr, int length)
+{
+	(void)snprintf(cpustr, length, "CPU:");
+	cpumask_scnprintf(cpustr + strlen(cpustr), length - strlen(cpustr), banned_cpus);
+}
+
+static void fill_banned_irqs(char *irqstr, int length)
+{
+	GList *entry = NULL;
+	GList *next = NULL;
+	struct irq_info *info = NULL;
+
+	(void)snprintf(irqstr, length, "IRQ:");
+	entry = g_list_first(cl_banned_irqs);
+	if (!entry)
+		(void)snprintf(irqstr + strlen(irqstr), length - strlen(irqstr), "None ");
+	while (entry) {
+		next = g_list_next(entry);
+		info = entry->data;
+		(void)snprintf(irqstr + strlen(irqstr), length - strlen(irqstr),
+				"%d,", info->irq);
+		entry = next;
+	}
+
+	(void)snprintf(irqstr + strlen(irqstr) - 1, length - strlen(irqstr), " PCI:");
+
+	entry = g_list_first(vm_banned_irqs);
+	if (!entry)
+		(void)snprintf(irqstr + strlen(irqstr), length - strlen(irqstr), "None");
+	while (entry) {
+		next = g_list_next(entry);
+		info = entry->data;
+		(void)snprintf(irqstr + strlen(irqstr), length - strlen(irqstr),
+				"%d,", info->irq);
+		entry = next;
+	}
+}
+
+static int send_msg(int fd, char* buf, int len)
+{
+	int ret = -1;
+
+	do {
+		ret = send(fd, buf, len, 0);
+	} while ((ret <= 0) && (errno == EINTR));
+
+	return ret;
+}
+
+static int setnonblocking(int fd)
+{
+	if (fcntl(fd, F_SETFL,
+		(int)((unsigned int)fcntl(socket_fd, F_GETFL, 0) | O_NONBLOCK)) < 0) {
+		return -1;
+	}
+	return 0;
+}
+
+static void prase_banned_irq_string(char *irq_string)
+{
+	char *end = NULL;
+	char *last = irq_string;
+	int irq;
+
+	g_list_free_full(cl_banned_irqs, free);
+	cl_banned_irqs = NULL;
+
+	log(TO_ALL, LOG_INFO, "Ban irqs(%s) online.\n", irq_string);
+	if (!strncmp(irq_string, "NONE", strlen("NONE"))) {
+		return;
+	}
+
+	irq = strtoul(irq_string, &end, 10);
+	while (last != end) {
+		add_cl_banned_irq(irq);
+		log(TO_ALL, LOG_INFO, "add banned irq %d from api.\n", irq);
+		last = end;
+		irq = strtoul(end, &end, 10);
+	};
+}
+
+static void prase_banned_cpu_string(char *cpu_ban_string)
+{
+	if (strlen(cpu_ban_string) < 1)
+		return;
+	if (!is_set_banned_cpumask_from_ui)
+		is_set_banned_cpumask_from_ui = 1;
+
+	strcpy(banned_cpumask_from_ui, cpu_ban_string);
+	log(TO_ALL, LOG_INFO, "Ban cpus(%s) online.\n", cpu_ban_string);
+	if (!strncmp(banned_cpumask_from_ui, "NONE", strlen("NONE"))) {
+		memset((void*)banned_cpumask_from_ui, 0, NR_CPUS + 1);
+	}
+}
+
+gboolean recv_msg_handle(gint client_fd,
+			GIOCondition condition,
+			gpointer user_data __attribute__((unused)))
+{
+	char buff[FILE_LINE_MAX_NUM];
+	struct cmsghdr *cmsg;
+	int recv_size = 0;
+	int valid_user = 0;
+	struct iovec iov = { buff, FILE_LINE_MAX_NUM - 1 };
+	struct msghdr msg = { 0 };
+	int new_iterval = 0;
+
+	if (condition == G_IO_IN) {
+		msg.msg_iov = &iov;
+		msg.msg_iovlen = 1;
+		msg.msg_control = malloc(CMSG_SPACE(sizeof(struct ucred)));
+		if (!msg.msg_control) {
+			goto out_close;
+		}
+		msg.msg_controllen = CMSG_SPACE(sizeof(struct ucred));
+		if ((recv_size = recvmsg(client_fd, &msg, 0)) < 0) {
+			log(TO_ALL, LOG_WARNING, "Error while receiving data:%d %s\n", recv_size, strerror(errno));
+			goto out_close;
+		}
+		cmsg = CMSG_FIRSTHDR(&msg);
+		if ((cmsg->cmsg_level == SOL_SOCKET) &&
+				(cmsg->cmsg_type == SCM_CREDENTIALS)) {
+			struct ucred *credentials = (struct ucred *) CMSG_DATA(cmsg);
+			if (!credentials->uid) {
+				valid_user = 1;
+			}
+		}
+		if (!valid_user) {
+			log(TO_ALL, LOG_INFO, "Permission denied for user to connect to socket.\n");
+			goto out_close;
+		}
+		if (!strncmp(buff, "settings ", strlen("settings "))) {
+			if (!(strncmp(buff + strlen("settings "), "sleep ",
+							strlen("sleep ")))) {
+				new_iterval = strtoul(buff + strlen("settings sleep "), NULL, 10);
+				if (new_iterval >= 1) {
+					sleep_interval = new_iterval;
+				}
+			} else if (!(strncmp(buff + strlen("settings "), "ban irqs \"",
+							strlen("ban irqs \"")))
+					&& buff[recv_size - 1] == '\"') {
+				buff[recv_size - 1] = '\0';
+				prase_banned_irq_string(buff + strlen("settings ban irqs \""));
+				need_rescan = 1;
+			} else if (!(strncmp(buff + strlen("settings "), "cpus ",
+							strlen("cpus")))) {
+				if (recv_size > (int)(NR_CPUS + strlen("settings cpus "))) {
+					log(TO_ALL, LOG_WARNING, "Setting cpus more than limit: %d.\n",
+						NR_CPUS);
+					goto out_close;
+				}
+				buff[recv_size] = '\0';
+				prase_banned_cpu_string(buff + strlen("settings cpus "));
+				need_rescan = 1;
+			}
+		} else if (!strncmp(buff, "stats ", strlen("stats "))) {
+			if (!(strncmp(buff + strlen("stats "), "cpu", strlen("cpu")))) {
+				fill_banned_cpus(buff, FILE_LINE_MAX_NUM - 1);
+				send_msg(client_fd, buff, strlen(buff));
+			} else if (!(strncmp(buff + strlen("stats "), "irq", strlen("irq")))) {
+				fill_banned_irqs(buff, FILE_LINE_MAX_NUM - 1);
+				send_msg(client_fd, buff, strlen(buff));
+			}
+		}
+out_close:
+		close(client_fd);
+		free(msg.msg_control);
+		cur_fd_count--;
+	}
+	return FALSE;
+}
+
+gboolean accept_handle(gint fd,
+			GIOCondition condition,
+			gpointer user_data __attribute__((unused)))
+{
+	int client_fd;
+
+	if (condition != G_IO_IN) {
+		return TRUE;
+	}
+	client_fd = accept(fd, NULL, NULL);
+	if (client_fd < 0) {
+		log(TO_ALL, LOG_WARNING, "Connection couldn't be accepted.\n");
+		return TRUE;
+	}
+	if (cur_fd_count >= MAX_EVENTS) {
+		log(TO_ALL, LOG_WARNING, "Daemon support no more than %d connection\n",
+			MAX_EVENTS);
+		close(client_fd);
+		return TRUE;
+	}
+	if (setnonblocking(client_fd) < 0) {
+		log(TO_ALL, LOG_WARNING, "Daemon set connection nonblocking failed: %s.\n",
+			strerror(errno));
+		close(client_fd);
+		return TRUE;
+	}
+	cur_fd_count++;
+	g_unix_fd_add(client_fd, G_IO_IN, recv_msg_handle, NULL);
+	return TRUE;
+}
+
+
+int init_unblock_socket()
+{
+	struct sockaddr_un addr;
+	int optval = 1;
+
+	(void)unlink(socket_name);
+
+	socket_fd = socket(AF_LOCAL, SOCK_STREAM, 0);
+	if (socket_fd < 0) {
+		log(TO_ALL, LOG_WARNING, "Socket couldn't be created.\n");
+		return 1;
+	}
+
+	/*
+	 * First try to create a file-based socket in tmpfs.  If that doesn't
+	 * succeed, fall back to an abstract socket (non file-based).
+	 */
+	memset(&addr, 0, sizeof(struct sockaddr_un));
+	addr.sun_family = AF_UNIX;
+	snprintf(socket_name, 64, "%s/%s%d.sock", SOCKET_TMPFS, SOCKET_PATH, getpid());
+	strncpy(addr.sun_path, socket_name, sizeof(addr.sun_path));
+	if (bind(socket_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
+		log(TO_ALL, LOG_WARNING, "Daemon couldn't be bound to the file-based socket.\n");
+		/* Try binding to abstract */
+		memset(&addr, 0, sizeof(struct sockaddr_un));
+		addr.sun_family = AF_UNIX;
+		if (bind(socket_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
+			log(TO_ALL, LOG_WARNING, "Daemon couldn't be bound to the abstract socket.\n");
+			return 1;
+		}
+	}
+
+	if (chmod(socket_name, SOCK_RWX_MODE) != 0) {
+		/* when use abstract socket, chmod may fail. No need return here. */
+		log(TO_ALL, LOG_WARNING, "socket name : %s, chmod failed\n", socket_name);
+	}
+
+	if (setsockopt(socket_fd, SOL_SOCKET, SO_PASSCRED, &optval, sizeof(optval)) < 0) {
+		log(TO_ALL, LOG_WARNING, "Unable to set socket options: %s.\n", strerror(errno));
+		return 1;
+	}
+
+	if (listen(socket_fd, MAX_CLIENT) != 0) {
+		log(TO_ALL, LOG_WARNING, "Daemon couldn't be listening to the socket: %s.\n",
+			strerror(errno));
+		return 1;
+	}
+
+	if (setnonblocking(socket_fd) < 0) {
+		log(TO_ALL, LOG_WARNING, "Daemon set connection(listen) nonblocking failed: %s.\n",
+			strerror(errno));
+		return 1;
+	}
+	banned_cpumask_from_ui = (char*)malloc(NR_CPUS + 1);
+	if (!banned_cpumask_from_ui) {
+		log(TO_ALL, LOG_WARNING, "Daemon faild to malloc banned_cpumask_from_ui space.\n");
+		return 1;
+	}
+	memset((void*)banned_cpumask_from_ui, 0, NR_CPUS + 1);
+
+	g_unix_fd_add(socket_fd, G_IO_IN, accept_handle, NULL);
+
+	return 0;
+}
diff --git a/sockapi.h b/sockapi.h
new file mode 100644
index 0000000..3875234
--- /dev/null
+++ b/sockapi.h
@@ -0,0 +1,19 @@
+#ifndef UVP_IRQBALANCE_SOCKAPI_H_
+#define UVP_IRQBALANCE_SOCKAPI_H_
+
+#include <sys/stat.h>
+
+/* set socket file mode to 660 */
+#define SOCK_RWX_MODE  (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP)
+
+#define SLEEP_TIME                  (100 * 1000)    /* 100ms */
+
+#define MAX_CLIENT                  512
+
+#define MAX_EVENTS                  1024
+
+#define FILE_LINE_MAX_NUM           (4096*6)
+
+int init_unblock_socket();
+
+#endif
-- 
1.8.3.1