irqbalance/bugfix-force-irq-into-rebalance-list-when-irq-removed-and-reinserted.patch

From a501662e98e2937cb63f3308d6497e723f838238 Mon Sep 17 00:00:00 2001
From: liuchao <liuchao173@huawei.com>
Date: Wed, 18 Mar 2020 22:08:33 +0800
Subject: [PATCH] force irq into rebalance list when irq removed and reinserted

prevent irq may be inserted to rebalance list more than once and add one msi irq
at one time to prevent new msi irqs stay on one numa node

---
 classify.c       | 154 +++++++++++++++++++++++++++++++++++++++++++++++++------
 irqbalance.c     |   6 ++-
 irqbalance.h     |   7 +++
 procinterrupts.c | 145 ++++++++++++++++++++++++++++++++++-----------------
 types.h          |   1 +
 5 files changed, 250 insertions(+), 63 deletions(-)

diff --git a/classify.c b/classify.c
index b40fcc1..2dc93ca 100644
--- a/classify.c
+++ b/classify.c
@@ -37,6 +37,7 @@ static GList *interrupts_dbs = NULL;
 static GList *banned_irqs = NULL;
 GList *cl_banned_irqs = NULL;
 static GList *cl_banned_modules = NULL;
+extern int need_add_single;
 
 #define SYSFS_DIR "/sys"
 #define SYSPCI_DIR "/sys/bus/pci/devices"
@@ -259,7 +259,7 @@
 	return irq_class;
 }
 
-static gint compare_ints(gconstpointer a, gconstpointer b)
+gint compare_ints(gconstpointer a, gconstpointer b)
 {
 	const struct irq_info *ai = a;
 	const struct irq_info *bi = b;
@@ -584,11 +585,13 @@ static int check_for_irq_ban(char *path __attribute__((unused)), int irq, GList
 	/*
 	 * Check to see if we banned module which the irq belongs to.
 	 */
-	entry = g_list_find_custom(proc_interrupts, &find, compare_ints);
-	if (entry) {
-		res = entry->data;
-		if (check_for_module_ban(res->name))
-			return 1;
+	if (proc_interrupts) {
+		entry = g_list_find_custom(proc_interrupts, &find, compare_ints);
+		if (entry) {
+			res = entry->data;
+			if (check_for_module_ban(res->name))
+				return 1;
+		}
 	}
 
 #ifdef INCLUDE_BANSCRIPT
@@ -624,16 +627,32 @@ static int check_for_irq_ban(char *path __attribute__((unused)), int irq, GList
 	return 0;
 }
 
+int is_proc_irq_info_exist(int irq, GList *tmp_list)
+{
+	GList *entry;
+	struct irq_info find;
+
+	if (!tmp_list) {
+		return 1;
+	}
+
+	find.irq = irq;
+	entry = g_list_find_custom(tmp_list, &find, compare_ints);
+
+	return entry ? 1 : 0;
+}
+
 /*
  * Figures out which interrupt(s) relate to the device we"re looking at in dirname
  */
-static void build_one_dev_entry(const char *dirname, GList *tmp_irqs)
+struct irq_info *build_one_dev_entry(const char *dirname, GList *tmp_list)
 {
 	struct dirent *entry;
 	DIR *msidir;
 	FILE *fd;
 	int irqnum;
-	struct irq_info *new, hint;
+	struct irq_info *new = NULL;
+	struct irq_info hint;
 	char path[PATH_MAX];
 	char devpath[PATH_MAX];
 	struct user_irq_policy pol;
@@ -657,10 +676,16 @@ static void build_one_dev_entry(const char *dirname, GList *tmp_irqs)
 				if (new)
 					continue;
 				get_irq_user_policy(devpath, irqnum, &pol);
-				if ((pol.ban == 1) || (check_for_irq_ban(devpath, irqnum, tmp_irqs))) {
+				if ((pol.ban == 1) || (check_for_irq_ban(devpath, irqnum, tmp_list))) {
 					__add_banned_irq(irqnum, &banned_irqs);
 					continue;
 				}
+				if (!is_proc_irq_info_exist(irqnum, tmp_list)) {
+					continue;
+				}
+				if (need_add_single && need_add_single != irqnum) {
+					continue;
+				}
 				hint.irq = irqnum;
 				hint.type = IRQ_TYPE_MSIX;
 				new = add_one_irq_to_db(devpath, &hint, &pol);
@@ -669,13 +694,13 @@ static void build_one_dev_entry(const char *dirname, GList *tmp_irqs)
 			}
 		} while (entry != NULL);
 		closedir(msidir);
-		return;
+		return new;
 	}
 
 	sprintf(path, "%s/%s/irq", SYSPCI_DIR, dirname);
 	fd = fopen(path, "r");
 	if (!fd)
-		return;
+		return new;
 	if (fscanf(fd, "%d", &irqnum) < 0)
 		goto done;
 
@@ -692,10 +717,13 @@ static void build_one_dev_entry(const char *dirname, GList *tmp_irqs)
 		if (new)
 			goto done;
 		get_irq_user_policy(devpath, irqnum, &pol);
-		if ((pol.ban == 1) || (check_for_irq_ban(devpath, irqnum, tmp_irqs))) {
+		if ((pol.ban == 1) || (check_for_irq_ban(devpath, irqnum, tmp_list))) {
 			__add_banned_irq(irqnum, &banned_irqs);
 			goto done;
 		}
+		if (!is_proc_irq_info_exist(irqnum, tmp_list)) {
+			goto done;
+		}
 
 		hint.irq = irqnum;
 		hint.type = IRQ_TYPE_LEGACY;
@@ -706,7 +734,60 @@ static void build_one_dev_entry(const char *dirname, GList *tmp_irqs)
 
 done:
 	fclose(fd);
-	return;
+	return new;
+}
+
+void find_irq_dev_path(int irq, char *dirname, int length)
+{
+	char cmd[PATH_MAX + 128];
+	FILE *output = NULL;
+	char path[PATH_MAX];
+	char buffer[128];
+	char *brc = NULL;
+	size_t dirlen;
+
+	memset(dirname, 0, length);
+	/* Return defaults if irq is 0 */
+	if (!irq)
+		return;
+
+	sprintf(path, "%s/*/msi_irqs", SYSPCI_DIR);
+	sprintf(cmd, "exec find %s -type f -name %d | awk -F '/' '{print $6}' ", path, irq);
+	output = popen(cmd, "r");
+	if (!output) {
+		log(TO_CONSOLE, LOG_WARNING, "Unable to execute IRQ %d path %s\n", irq, path);
+		return;
+	}
+
+	brc = fgets(buffer, 128, output);
+	/* fgets will get a redundant \n */
+	if (brc && (dirlen = strcspn(brc, "\n")) > 0) {
+		log(TO_CONSOLE, LOG_INFO, "msi_irqs IRQ %d dirname is %s\n", irq, brc);
+		brc[dirlen] = '\0';
+		strncpy(dirname, brc, length);
+		pclose(output);
+		return;
+	}
+	pclose(output);
+
+	sprintf(path, "%s/*/irq", SYSPCI_DIR);
+	sprintf(cmd, "exec grep -w %d %s | awk -F '/' '{print $6}' ", irq, path);
+	output = popen(cmd, "r");
+	if (!output) {
+		log(TO_CONSOLE, LOG_WARNING, "Unable to execute IRQ %d path %s\n", irq, path);
+		return;
+	}
+
+	brc = fgets(buffer, 128, output);
+	if (brc && (dirlen = strcspn(brc, "\n")) > 0) {
+		log(TO_CONSOLE, LOG_INFO, "IRQ %d dirname is %s\n", irq, brc);
+		brc[dirlen] = '\0';
+		strncpy(dirname, brc, length);
+		pclose(output);
+		return;
+	}
+	pclose(output);
+
 }
 
 static void free_irq(struct irq_info *info, void *data __attribute__((unused)))
@@ -714,6 +795,45 @@ static void free_irq(struct irq_info *info, void *data __attribute__((unused)))
 	free(info);
 }
 
+static void remove_no_existing_irq(struct irq_info *info, void *data __attribute__((unused)))
+{
+	GList *entry = NULL;
+
+	if (info->existing) {
+		info->existing = 0;
+		return;
+	}
+
+	entry = g_list_find_custom(interrupts_db, info, compare_ints);
+	if (entry)
+		interrupts_db = g_list_delete_link(interrupts_db, entry);
+
+	entry = g_list_find_custom(banned_irqs, info, compare_ints);
+	if (entry)
+		banned_irqs = g_list_delete_link(banned_irqs, entry);
+
+	entry = g_list_find_custom(rebalance_irq_list, info, compare_ints);
+	if (entry)
+		rebalance_irq_list = g_list_delete_link(rebalance_irq_list, entry);
+
+	if(info->assigned_obj) {
+		entry = g_list_find_custom(info->assigned_obj->interrupts, info, compare_ints);
+	    if (entry) {
+			info->assigned_obj->interrupts = g_list_delete_link(info->assigned_obj->interrupts, entry);
+		}
+	}
+	log(TO_CONSOLE, LOG_INFO, "IRQ %d is removed from interrupts_db.\n", info->irq);
+	free_irq(info, NULL);
+}
+
+void clear_no_existing_irqs(void)
+{
+	for_each_irq(NULL, remove_no_existing_irq, NULL);
+	if (banned_irqs){
+		for_each_irq(banned_irqs, remove_no_existing_irq, NULL);
+	}
+}
+
 void free_irq_db(void)
 {
 	for_each_irq(NULL, free_irq, NULL);
@@ -733,14 +853,14 @@ void free_cl_opts(void)
 	g_list_free(banned_irqs);
 }
 
-static void add_new_irq(int irq, struct irq_info *hint, GList *proc_interrupts)
+struct irq_info *add_new_irq(int irq, struct irq_info *hint, GList *proc_interrupts)
 {
-	struct irq_info *new;
+	struct irq_info *new = NULL;
 	struct user_irq_policy pol;
 
 	new = get_irq_info(irq);
 	if (new)
-		return;
+		return new;
 
 	/* Set NULL devpath for the irq has no sysfs entries */
 	get_irq_user_policy(NULL, irq, &pol);
@@ -752,6 +872,8 @@ static void add_new_irq(int irq, struct irq_info *hint, GList *proc_interrupts)
 
 	if (!new)
 		log(TO_CONSOLE, LOG_WARNING, "add_new_irq: Failed to add irq %d\n", irq);
+
+	return new;
 }
 
 static void add_missing_irq(struct irq_info *info, void *attr)
diff --git a/irqbalance.c b/irqbalance.c
index f965a2a..4d0a417 100644
--- a/irqbalance.c
+++ b/irqbalance.c
@@ -251,11 +251,15 @@ static void dump_object_tree(void)
 	for_each_object(numa_nodes, dump_numa_node_info, NULL);
 }
 
-static void force_rebalance_irq(struct irq_info *info, void *data __attribute__((unused)))
+void force_rebalance_irq(struct irq_info *info, void *data __attribute__((unused)))
 {
 	if (info->level == BALANCE_NONE)
 		return;
 
+	/* Prevent inserting a duplicate entry to avoid list chaos */
+	if (g_list_find_custom(rebalance_irq_list, info, compare_ints))
+		return;
+
 	if (info->assigned_obj == NULL)
 		rebalance_irq_list = g_list_append(rebalance_irq_list, info);
 	else
diff --git a/irqbalance.h b/irqbalance.h
index 3a78c7f..9e28285 100644
--- a/irqbalance.h
+++ b/irqbalance.h
@@ -109,6 +109,13 @@ extern void add_banned_irq(int irq);
 extern void remove_one_irq_from_db(int irq);
 #define irq_numa_node(irq) ((irq)->numa_node)
 
+extern struct irq_info *build_one_dev_entry(const char *dirname, GList *tmp_list);
+extern void find_irq_dev_path(int irq, char *dirname, int length);
+extern struct irq_info *add_new_irq(int irq, struct irq_info *hint, GList *proc_interrupts);
+extern void clear_no_existing_irqs(void);
+extern void force_rebalance_irq(struct irq_info *info, void *data __attribute__((unused)));
+extern gint compare_ints(gconstpointer a, gconstpointer b);
+
 extern unsigned long migrate_val;
 extern unsigned long load_limit;
 /*
diff --git a/procinterrupts.c b/procinterrupts.c
index 70831b4..358458c 100644
--- a/procinterrupts.c
+++ b/procinterrupts.c
@@ -42,6 +42,7 @@
 
 static int proc_int_has_msi = 0;
 static int msi_found_in_sysfs = 0;
+int need_add_single = 0;
 
 #ifdef AARCH64
 struct irq_match {
@@ -144,6 +145,52 @@ static void guess_arm_irq_hints(char *name, struct irq_info *info)
 	
 }
 #endif
+static void init_irq_class_and_type(char *savedline, struct irq_info *info, int irq) {
+	char *irq_name = NULL;
+	char *irq_mod = NULL;
+	char *savedptr = NULL;
+	char *last_token = NULL;
+	char *p = NULL;
+	int	is_xen_dyn = 0;
+#ifdef AARCH64
+	char *tmp = NULL;
+#endif
+
+	irq_name = strtok_r(savedline, " ", &savedptr);
+	if (strstr(irq_name, "xen-dyn") != NULL)
+		is_xen_dyn = 1;
+	last_token = strtok_r(NULL, " ", &savedptr);
+	while ((p = strtok_r(NULL, " ", &savedptr))) {
+		irq_name = last_token;
+		if (strstr(irq_name, "xen-dyn") != NULL)
+			is_xen_dyn = 1;
+		last_token = p;
+	}
+
+#ifdef AARCH64
+	/* Of course the formatting for /proc/interrupts is different on different arches */
+	irq_name = last_token;
+	tmp = strchr(irq_name, '\n');
+	if (tmp)
+		*tmp = 0;
+#endif
+	irq_mod = last_token;
+	info->irq = irq;
+
+	if (strstr(irq_name, "-event") != NULL && is_xen_dyn == 1) {
+		info->type = IRQ_TYPE_VIRT_EVENT;
+		info->class = IRQ_VIRT_EVENT;
+	} else {
+#ifdef AARCH64
+		guess_arm_irq_hints(irq_name, info);
+#else
+		info->type = IRQ_TYPE_LEGACY;
+		info->class = IRQ_OTHER;
+#endif
+	}
+	info->name = strdupa(irq_mod);
+}
+
 
 GList* collect_full_irq_list()
 {
@@ -151,10 +198,6 @@ GList* collect_full_irq_list()
 	FILE *file;
 	char *line = NULL;
 	size_t size = 0;
-	char *irq_name, *irq_mod, *savedptr, *last_token, *p;
-#ifdef AARCH64
-	char *tmp;
-#endif
 
 	file = fopen("/proc/interrupts", "r");
 	if (!file)
@@ -169,7 +212,6 @@ GList* collect_full_irq_list()
 
 	while (!feof(file)) {
 		int	 number;
-		int      is_xen_dyn = 0;
 		struct irq_info *info;
 		char *c;
 		char *savedline = NULL;
@@ -191,45 +233,13 @@ GList* collect_full_irq_list()
 		savedline = strdup(line);
 		if (!savedline)
 			break;
-		irq_name = strtok_r(savedline, " ", &savedptr);
-		if (strstr(irq_name, "xen-dyn") != NULL)
-			is_xen_dyn = 1;
-		last_token = strtok_r(NULL, " ", &savedptr);
-		while ((p = strtok_r(NULL, " ", &savedptr))) {
-			irq_name = last_token;
-			if (strstr(irq_name, "xen-dyn") != NULL)
-				is_xen_dyn = 1;
-			last_token = p;
-		}
-
-#ifdef AARCH64
-		/* Of course the formatting for /proc/interrupts is different on different arches */
-		irq_name = last_token;
-		tmp = strchr(irq_name, '\n');
-		if (tmp)
-			*tmp = 0;
-#endif
-		irq_mod = last_token;
-
 		*c = 0;
 		c++;
 		number = strtoul(line, NULL, 10);
 
 		info = calloc(sizeof(struct irq_info), 1);
 		if (info) {
-			info->irq = number;
-			if (strstr(irq_name, "-event") != NULL && is_xen_dyn == 1) {
-				info->type = IRQ_TYPE_VIRT_EVENT;
-				info->class = IRQ_VIRT_EVENT;
-			} else {
-#ifdef AARCH64
-				guess_arm_irq_hints(irq_name, info);
-#else
-				info->type = IRQ_TYPE_LEGACY;
-				info->class = IRQ_OTHER;
-#endif
-			}
-			info->name = strdup(irq_mod);
+			init_irq_class_and_type(savedline, info, number);
 			tmp_list = g_list_append(tmp_list, info);
 		}
 		free(savedline);
@@ -239,6 +249,14 @@ GList* collect_full_irq_list()
 	return tmp_list;
 }
 
+/* parsing /proc/interrrupts to detect whether removed and reinserted IRQ
+* device happened or not. If yes, then IRQs have to be rescanning again;
+* However, in order to keep no impact on online running IRQs performance stable,
+* removed and reinserted IRQ added back into rebalance_irq_list,
+* for irq load re-calculation instead of trigger rescanning all IRQs.
+* specially, when a new IRQ is detected, it has to be checked what devpath is,
+* then it is added into database accordingly.
+*/
 void parse_proc_interrupts(void)
 {
 	FILE *file;
@@ -262,7 +280,10 @@ void parse_proc_interrupts(void)
 		uint64_t count;
 		char *c, *c2;
 		struct irq_info *info;
-		char savedline[1024];
+		struct irq_info tmp_info = {0};
+		char *savedline = NULL;
+		char dirname[PATH_MAX] = {'\0'};
+		struct irq_info *lookup;
 
 		if (getline(&line, &size, file)<=0)
 			break;
@@ -282,7 +303,9 @@ void parse_proc_interrupts(void)
 		if (!c)
 			continue;
 
-		strncpy(savedline, line, sizeof(savedline)-1);
+		savedline = strdup(line);
+		if (!savedline)
+			continue;
 
 		*c = 0;
 		c++;
@@ -290,9 +313,37 @@ void parse_proc_interrupts(void)
 
 		info = get_irq_info(number);
 		if (!info) {
-			need_rescan = 1;
-			break;
+			init_irq_class_and_type(savedline, &tmp_info, number);
+			find_irq_dev_path(number, dirname, PATH_MAX);
+			if (strlen(dirname) > 0) {
+				need_add_single = number;
+				info = build_one_dev_entry(dirname, NULL);
+				need_add_single = 0;
+				lookup = get_irq_info(number);
+				if (lookup != NULL) {
+					lookup->existing = 1;
+					info = lookup;
+				}
+				log(TO_CONSOLE, LOG_INFO, "new IRQ %d added into database, dirname %s\n", number, dirname);
+			} else {
+				info = add_new_irq(number, &tmp_info, NULL);
+			}
+			if (tmp_info.name) {
+				free(tmp_info.name);
+				tmp_info.name = NULL;
+			}
+
+			if (info) {
+				force_rebalance_irq(info, NULL);
+				log(TO_CONSOLE, LOG_INFO, "new IRQ %d added into rebalance list\n", number);
+			} else {
+				need_rescan = 1;
+				free(savedline);
+				break;
+			}
 		}
+		free(savedline);
+		info->existing = 1;
 
 		count = 0;
 		cpunr = 0;
@@ -316,17 +367,19 @@ void parse_proc_interrupts(void)
 		 * cause an overflow and IRQ won't be rebalanced again
 		 */
 		if (count < info->irq_count) {
-			need_rescan = 1;
-			break;
+			log(TO_CONSOLE, LOG_INFO, "Removed and reinserted IRQ %d added into rebalance list\n", number);
+			force_rebalance_irq(info, NULL);
 		}
 
-		info->last_irq_count = info->irq_count;		
+		info->last_irq_count = info->irq_count;
 		info->irq_count = count;
 
 		/* is interrupt MSI based? */
 		if ((info->type == IRQ_TYPE_MSI) || (info->type == IRQ_TYPE_MSIX))
 			msi_found_in_sysfs = 1;
-	}		
+	}
+	clear_no_existing_irqs();
+
 	if ((proc_int_has_msi) && (!msi_found_in_sysfs) && (!need_rescan)) {
 		log(TO_ALL, LOG_WARNING, "WARNING: MSI interrupts found in /proc/interrupts\n");
 		log(TO_ALL, LOG_WARNING, "But none found in sysfs, you need to update your kernel\n");
diff --git a/types.h b/types.h
index a01d649..9693cf4 100644
--- a/types.h
+++ b/types.h
@@ -70,6 +70,7 @@ struct irq_info {
 	uint64_t last_irq_count;
 	uint64_t load;
 	int moved;
+	int existing;
 	struct topo_obj *assigned_obj;
 	char *name;
 };
-- 
1.8.3.1