irqbalance/feature-irqbalance-auto-banned-pci-assigned-irq.patch

From fd447c6f129769fcb685d8a2e22c75a922efe342 Mon Sep 17 00:00:00 2001
From: Zengruan Ye <yezengruan@huawei.com>
Date: Mon, 15 Jul 2019 21:35:29 +0800
Subject: [PATCH 3/6] feature: irqbalance: auto banned pci-assigned irq

checkout VM pci-assigned pci device irq, and banned it

Signed-off-by: Xu Yandong <xuyandong2@huawei.com>
---
 activate.c       | 38 ++++++++++++++++++++++++++++++++++++-
 classify.c       | 58 +++++++++++++++++++++++++++++++++++++++++++++++++-------
 irqbalance.c     | 11 +++++++++++
 irqbalance.h     |  2 ++
 procinterrupts.c | 23 ++++++++++++++++++++++
 types.h          |  1 +
 6 files changed, 125 insertions(+), 8 deletions(-)

diff --git a/activate.c b/activate.c
index 1c4b867..ad60fde 100644
--- a/activate.c
+++ b/activate.c
@@ -29,9 +29,12 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <stdint.h>
+#include <dirent.h>
 
 #include "irqbalance.h"
 
+extern int ban_pci_assigned_irq;
+
 static int check_affinity(struct irq_info *info, cpumask_t applied_mask)
 {
 	cpumask_t current_mask;
@@ -56,6 +59,30 @@ static int check_affinity(struct irq_info *info, cpumask_t applied_mask)
 	return cpus_equal(applied_mask, current_mask);
 }
 
+static int is_still_pci_assigned_irq(int irq)
+{
+	DIR *dir = NULL;
+	struct dirent *ptr = NULL;
+	char irq_path[PATH_MAX] = { 0 };
+
+	snprintf(irq_path, PATH_MAX - 1, "/proc/irq/%i/", irq);
+	if ((dir = opendir(irq_path)) == NULL) {
+		return 0;
+	}
+
+	while ((ptr = readdir(dir)) != NULL) {
+		if(strcmp(ptr->d_name, ".") == 0 || strcmp(ptr->d_name, "..") == 0) {
+			continue;
+		} else if (is_pci_assigned_irq(ptr->d_name)) {
+			closedir(dir);
+			return 1;
+		}
+	}
+
+	closedir(dir);
+	return 0;
+}
+
 static void activate_mapping(struct irq_info *info, void *data __attribute__((unused)))
 {
 	char buf[PATH_MAX];
@@ -89,7 +116,16 @@ static void activate_mapping(struct irq_info *info, void *data __attribute__((un
 		return;
 
 	cpumask_scnprintf(buf, PATH_MAX, applied_mask);
-	fprintf(file, "%s", buf);
+	if (ban_pci_assigned_irq) {
+		if (!is_still_pci_assigned_irq(info->irq)) {
+			fprintf(file, "%s", buf);
+		} else {
+			log(TO_CONSOLE, LOG_INFO, "IRQ %d is turned into a PCI-assigned irq number.\n", info->irq);
+			need_rescan = 1;
+		}
+	} else {
+		fprintf(file, "%s", buf);
+	}
 	fclose(file);
 	info->moved = 0; /*migration is done*/
 }
diff --git a/classify.c b/classify.c
index 37bfb29..52fd74a 100644
--- a/classify.c
+++ b/classify.c
@@ -37,6 +37,7 @@ static GList *interrupts_db = NULL;
 static GList *banned_irqs = NULL;
 GList *cl_banned_irqs = NULL;
 static GList *cl_banned_modules = NULL;
+static GList *vm_banned_irqs = NULL;
 
 #define SYSFS_DIR "/sys"
 #define SYSPCI_DIR "/sys/bus/pci/devices"
@@ -264,7 +265,7 @@ static gint compare_ints(gconstpointer a, gconstpointer b)
 	return ai->irq - bi->irq;
 }
 
-static void add_banned_irq(int irq, GList **list)
+static void add_banned_irq(int irq, GList **list, int extra_flag)
 {
 	struct irq_info find, *new;
 	GList *entry;
@@ -282,6 +283,7 @@ static void add_banned_irq(int irq, GList **list)
 
 	new->irq = irq;
 	new->flags |= IRQ_FLAG_BANNED;
+	new->flags |= extra_flag;
 
 	*list = g_list_append(*list, new);
 	log(TO_CONSOLE, LOG_INFO, "IRQ %d was BANNED.\n", irq);
@@ -291,13 +293,18 @@ static void add_banned_irq(int irq, GList **list)
 #ifdef AARCH64
 void add_banned_list_irq(int irq)
 {
-	add_banned_irq(irq, &banned_irqs);
+	add_banned_irq(irq, &banned_irqs, 0);
 }
 #endif
 
 void add_cl_banned_irq(int irq)
 {
-	add_banned_irq(irq, &cl_banned_irqs);
+	add_banned_irq(irq, &cl_banned_irqs, 0);
+}
+
+void add_vm_banned_irq(int irq)
+{
+	add_banned_irq(irq, &vm_banned_irqs, IRQ_FLAG_VM_BANNED);
 }
 
 static int is_banned_irq(int irq)
@@ -307,7 +314,10 @@ static int is_banned_irq(int irq)
 
 	find.irq = irq;
 
-	entry = g_list_find_custom(banned_irqs, &find, compare_ints);
+	entry = g_list_find_custom(vm_banned_irqs, &find, compare_ints);
+	if (!entry)
+		entry = g_list_find_custom(banned_irqs, &find, compare_ints);
+
 	return entry ? 1:0;
 }
 
@@ -660,7 +670,7 @@ struct irq_info *build_one_dev_entry(const char *dirname, GList *tmp_list)
 					continue;
 				get_irq_user_policy(devpath, irqnum, &pol);
 				if ((pol.ban == 1) || (check_for_irq_ban(devpath, irqnum, tmp_list))) {
-					add_banned_irq(irqnum, &banned_irqs);
+					add_banned_irq(irqnum, &banned_irqs, 0);
 					continue;
 				}
 				hint.irq = irqnum;
@@ -695,7 +705,7 @@ struct irq_info *build_one_dev_entry(const char *dirname, GList *tmp_list)
 			goto done;
 		get_irq_user_policy(devpath, irqnum, &pol);
 		if ((pol.ban == 1) || (check_for_irq_ban(path, irqnum, tmp_list))) {
-			add_banned_irq(irqnum, &banned_irqs);
+			add_banned_irq(irqnum, &banned_irqs, 0);
 			goto done;
 		}
 
@@ -811,6 +821,9 @@ void free_irq_db(void)
 	banned_irqs = NULL;
 	g_list_free(rebalance_irq_list);
 	rebalance_irq_list = NULL;
+	for_each_irq(vm_banned_irqs, free_irq, NULL);
+	g_list_free(vm_banned_irqs);
+	vm_banned_irqs = NULL;
 }
 
 void free_cl_opts(void)
@@ -832,7 +845,7 @@ struct irq_info *add_new_irq(int irq, struct irq_info *hint, GList *proc_interru
 	/* Set NULL devpath for the irq has no sysfs entries */
 	get_irq_user_policy(NULL, irq, &pol);
 	if ((pol.ban == 1) || check_for_irq_ban(NULL, irq, proc_interrupts)) { /*FIXME*/
-		add_banned_irq(irq, &banned_irqs);
+		add_banned_irq(irq, &banned_irqs, 0);
 		new = get_irq_info(irq);
 	} else
 		new = add_one_irq_to_db(NULL, hint, &pol);
@@ -906,6 +919,9 @@ struct irq_info *get_irq_info(int irq)
 	entry = g_list_find_custom(interrupts_db, &find, compare_ints);
 
 	if (!entry)
+		entry = g_list_find_custom(vm_banned_irqs, &find, compare_ints);
+
+	if (!entry)
 		entry = g_list_find_custom(banned_irqs, &find, compare_ints);
 
 	return entry ? entry->data : NULL;
@@ -954,3 +970,31 @@ void sort_irq_list(GList **list)
 {
 	*list = g_list_sort(*list, sort_irqs);
 }
+
+/*
+ * assigned pci device's irq in /proc/interrupts
+ * has keyword "vfio" or "kvm"
+ * and BDF notation "xxxx:xx:xx.x"
+ */
+int is_pci_assigned_irq(const char *line)
+{
+	const char *pos = NULL;
+
+	pos = strstr(line, "vfio");
+	if (!pos) {
+		pos = strstr(line, "kvm");
+	}
+
+	if (!pos)
+		return 0;
+
+	while (*pos) {
+		if (*pos >= '0' && *pos <= '9') {
+			if (*(pos + 4) == ':' && *(pos + 7) == ':' && *(pos + 10) == '.')
+				return 1;
+		}
+		pos++;
+	}
+
+	return 0;
+}
diff --git a/irqbalance.c b/irqbalance.c
index 4787cb2..40ec65c 100644
--- a/irqbalance.c
+++ b/irqbalance.c
@@ -44,6 +44,7 @@
 #include "irqbalance.h"
 
 volatile int keep_going = 1;
+volatile int ban_pci_assigned_irq = 1;
 int socket_fd;
 char socket_name[64];
 int one_shot_mode;
@@ -375,6 +376,12 @@ gboolean scan()
 	}
 }
 
+static void parse_ban_pci_assigned_irq(char *status)
+{
+	if (status && (0 == strcmp(status,"disabled")))
+		ban_pci_assigned_irq = 0;
+}
+
 void get_irq_data(struct irq_info *irq, void *data)
 {
 	char **irqdata = (char **)data;
@@ -619,6 +626,10 @@ int main(int argc, char** argv)
 		foreground_mode=1;
 	}
 
+	if (getenv("IRQBALANCE_AUTO_BAN_PCI_ASSIGNED_IRQS")) {
+		parse_ban_pci_assigned_irq(getenv("IRQBALANCE_AUTO_BAN_PCI_ASSIGNED_IRQS"));
+	}
+
 	/*
  	 * If we are't in debug mode, don't dump anything to the console
  	 * note that everything goes to the console before we check this
diff --git a/irqbalance.h b/irqbalance.h
index 5016cc8..339e2a3 100644
--- a/irqbalance.h
+++ b/irqbalance.h
@@ -103,8 +103,10 @@ extern void add_cl_banned_irq(int irq);
 #ifdef AARCH64
 extern void add_banned_list_irq(int irq);
 #endif
+extern void add_vm_banned_irq(int irq);
 extern void for_each_irq(GList *list, void (*cb)(struct irq_info *info,  void *data), void *data);
 extern struct irq_info *get_irq_info(int irq);
+extern int is_pci_assigned_irq(const char *line);
 extern void migrate_irq(GList **from, GList **to, struct irq_info *info);
 extern void free_cl_opts(void);
 extern void add_cl_banned_module(char *modname);
diff --git a/procinterrupts.c b/procinterrupts.c
index 6cfa661..522b9a1 100644
--- a/procinterrupts.c
+++ b/procinterrupts.c
@@ -42,6 +42,7 @@
 
 static int proc_int_has_msi = 0;
 static int msi_found_in_sysfs = 0;
+extern int ban_pci_assigned_irq;
 
 #ifdef AARCH64
 struct irq_match {
@@ -290,6 +291,12 @@ GList* collect_full_irq_list()
 		c++;
 		number = strtoul(line, NULL, 10);
 
+		if (ban_pci_assigned_irq && is_pci_assigned_irq(c)) {
+			log(TO_ALL, LOG_INFO, "Banned PCI-assigned irq %d.\n", number);
+			add_vm_banned_irq(number);
+			continue;
+		}
+
 #ifdef AARCH64
 		if (is_arm_irq_aff_cannot_change(number)) {
 			/*
@@ -333,6 +340,7 @@ void parse_proc_interrupts(void)
 	FILE *file;
 	char *line = NULL;
 	size_t size = 0;
+	int vfio_flag = 0;
 
 	file = fopen("/proc/interrupts", "r");
 	if (!file)
@@ -400,6 +408,21 @@ void parse_proc_interrupts(void)
 		}
 		info->existing = 1;
 
+		if (ban_pci_assigned_irq) {
+			vfio_flag = is_pci_assigned_irq(c);
+			if (vfio_flag && (info->flags & IRQ_FLAG_VM_BANNED) == 0) {
+				log(TO_ALL, LOG_INFO, "IRQ %d has been reused as a pci assigned interrupt number", number);
+				need_rescan = 1;
+				break;
+			}
+
+			if (!vfio_flag && (info->flags & IRQ_FLAG_VM_BANNED) != 0) {
+				log(TO_ALL, LOG_INFO, "IRQ %d has been reused as a general interrupt number", number);
+				need_rescan = 1;
+				break;
+			}
+		}
+
 		count = 0;
 		cpunr = 0;
 
diff --git a/types.h b/types.h
index 62cc2bb..c0950ee 100644
--- a/types.h
+++ b/types.h
@@ -35,6 +35,7 @@
  * IRQ Internal tracking flags
  */
 #define IRQ_FLAG_BANNED	1
+#define IRQ_FLAG_VM_BANNED  2
 
 enum obj_type_e {
 	OBJ_TYPE_CPU,
-- 
1.8.3.1