From fd447c6f129769fcb685d8a2e22c75a922efe342 Mon Sep 17 00:00:00 2001 From: Zengruan Ye Date: Mon, 15 Jul 2019 21:35:29 +0800 Subject: [PATCH 3/6] feature: irqbalance: auto banned pci-assigned irq checkout VM pci-assigned pci device irq, and banned it Signed-off-by: Xu Yandong --- activate.c | 38 ++++++++++++++++++++++++++++++++++++- classify.c | 58 +++++++++++++++++++++++++++++++++++++++++++++++++------- irqbalance.c | 11 +++++++++++ irqbalance.h | 2 ++ procinterrupts.c | 23 ++++++++++++++++++++++ types.h | 1 + 6 files changed, 125 insertions(+), 8 deletions(-) diff --git a/activate.c b/activate.c index 1c4b867..ad60fde 100644 --- a/activate.c +++ b/activate.c @@ -29,9 +29,12 @@ #include #include #include +#include #include "irqbalance.h" +extern int ban_pci_assigned_irq; + static int check_affinity(struct irq_info *info, cpumask_t applied_mask) { cpumask_t current_mask; @@ -56,6 +59,30 @@ static int check_affinity(struct irq_info *info, cpumask_t applied_mask) return cpus_equal(applied_mask, current_mask); } +static int is_still_pci_assigned_irq(int irq) +{ + DIR *dir = NULL; + struct dirent *ptr = NULL; + char irq_path[PATH_MAX] = { 0 }; + + snprintf(irq_path, PATH_MAX - 1, "/proc/irq/%i/", irq); + if ((dir = opendir(irq_path)) == NULL) { + return 0; + } + + while ((ptr = readdir(dir)) != NULL) { + if(strcmp(ptr->d_name, ".") == 0 || strcmp(ptr->d_name, "..") == 0) { + continue; + } else if (is_pci_assigned_irq(ptr->d_name)) { + closedir(dir); + return 1; + } + } + + closedir(dir); + return 0; +} + static void activate_mapping(struct irq_info *info, void *data __attribute__((unused))) { char buf[PATH_MAX]; @@ -89,7 +116,16 @@ static void activate_mapping(struct irq_info *info, void *data __attribute__((un return; cpumask_scnprintf(buf, PATH_MAX, applied_mask); - fprintf(file, "%s", buf); + if (ban_pci_assigned_irq) { + if (!is_still_pci_assigned_irq(info->irq)) { + fprintf(file, "%s", buf); + } else { + log(TO_CONSOLE, LOG_INFO, "IRQ %d is turned into a PCI-assigned irq number.\n", info->irq); + need_rescan = 1; + } + } else { + fprintf(file, "%s", buf); + } fclose(file); info->moved = 0; /*migration is done*/ } diff --git a/classify.c b/classify.c index 37bfb29..52fd74a 100644 --- a/classify.c +++ b/classify.c @@ -37,6 +37,7 @@ static GList *interrupts_db = NULL; static GList *banned_irqs = NULL; GList *cl_banned_irqs = NULL; static GList *cl_banned_modules = NULL; +static GList *vm_banned_irqs = NULL; #define SYSFS_DIR "/sys" #define SYSPCI_DIR "/sys/bus/pci/devices" @@ -264,7 +265,7 @@ static gint compare_ints(gconstpointer a, gconstpointer b) return ai->irq - bi->irq; } -static void add_banned_irq(int irq, GList **list) +static void add_banned_irq(int irq, GList **list, int extra_flag) { struct irq_info find, *new; GList *entry; @@ -282,6 +283,7 @@ static void add_banned_irq(int irq, GList **list) new->irq = irq; new->flags |= IRQ_FLAG_BANNED; + new->flags |= extra_flag; *list = g_list_append(*list, new); log(TO_CONSOLE, LOG_INFO, "IRQ %d was BANNED.\n", irq); @@ -291,13 +293,18 @@ static void add_banned_irq(int irq, GList **list) #ifdef AARCH64 void add_banned_list_irq(int irq) { - add_banned_irq(irq, &banned_irqs); + add_banned_irq(irq, &banned_irqs, 0); } #endif void add_cl_banned_irq(int irq) { - add_banned_irq(irq, &cl_banned_irqs); + add_banned_irq(irq, &cl_banned_irqs, 0); +} + +void add_vm_banned_irq(int irq) +{ + add_banned_irq(irq, &vm_banned_irqs, IRQ_FLAG_VM_BANNED); } static int is_banned_irq(int irq) @@ -307,7 +314,10 @@ static int is_banned_irq(int irq) find.irq = irq; - entry = g_list_find_custom(banned_irqs, &find, compare_ints); + entry = g_list_find_custom(vm_banned_irqs, &find, compare_ints); + if (!entry) + entry = g_list_find_custom(banned_irqs, &find, compare_ints); + return entry ? 1:0; } @@ -660,7 +670,7 @@ struct irq_info *build_one_dev_entry(const char *dirname, GList *tmp_list) continue; get_irq_user_policy(devpath, irqnum, &pol); if ((pol.ban == 1) || (check_for_irq_ban(devpath, irqnum, tmp_list))) { - add_banned_irq(irqnum, &banned_irqs); + add_banned_irq(irqnum, &banned_irqs, 0); continue; } hint.irq = irqnum; @@ -695,7 +705,7 @@ struct irq_info *build_one_dev_entry(const char *dirname, GList *tmp_list) goto done; get_irq_user_policy(devpath, irqnum, &pol); if ((pol.ban == 1) || (check_for_irq_ban(path, irqnum, tmp_list))) { - add_banned_irq(irqnum, &banned_irqs); + add_banned_irq(irqnum, &banned_irqs, 0); goto done; } @@ -811,6 +821,9 @@ void free_irq_db(void) banned_irqs = NULL; g_list_free(rebalance_irq_list); rebalance_irq_list = NULL; + for_each_irq(vm_banned_irqs, free_irq, NULL); + g_list_free(vm_banned_irqs); + vm_banned_irqs = NULL; } void free_cl_opts(void) @@ -832,7 +845,7 @@ struct irq_info *add_new_irq(int irq, struct irq_info *hint, GList *proc_interru /* Set NULL devpath for the irq has no sysfs entries */ get_irq_user_policy(NULL, irq, &pol); if ((pol.ban == 1) || check_for_irq_ban(NULL, irq, proc_interrupts)) { /*FIXME*/ - add_banned_irq(irq, &banned_irqs); + add_banned_irq(irq, &banned_irqs, 0); new = get_irq_info(irq); } else new = add_one_irq_to_db(NULL, hint, &pol); @@ -906,6 +919,9 @@ struct irq_info *get_irq_info(int irq) entry = g_list_find_custom(interrupts_db, &find, compare_ints); if (!entry) + entry = g_list_find_custom(vm_banned_irqs, &find, compare_ints); + + if (!entry) entry = g_list_find_custom(banned_irqs, &find, compare_ints); return entry ? entry->data : NULL; @@ -954,3 +970,31 @@ void sort_irq_list(GList **list) { *list = g_list_sort(*list, sort_irqs); } + +/* + * assigned pci device's irq in /proc/interrupts + * has keyword "vfio" or "kvm" + * and BDF notation "xxxx:xx:xx.x" + */ +int is_pci_assigned_irq(const char *line) +{ + const char *pos = NULL; + + pos = strstr(line, "vfio"); + if (!pos) { + pos = strstr(line, "kvm"); + } + + if (!pos) + return 0; + + while (*pos) { + if (*pos >= '0' && *pos <= '9') { + if (*(pos + 4) == ':' && *(pos + 7) == ':' && *(pos + 10) == '.') + return 1; + } + pos++; + } + + return 0; +} diff --git a/irqbalance.c b/irqbalance.c index 4787cb2..40ec65c 100644 --- a/irqbalance.c +++ b/irqbalance.c @@ -44,6 +44,7 @@ #include "irqbalance.h" volatile int keep_going = 1; +volatile int ban_pci_assigned_irq = 1; int socket_fd; char socket_name[64]; int one_shot_mode; @@ -375,6 +376,12 @@ gboolean scan() } } +static void parse_ban_pci_assigned_irq(char *status) +{ + if (status && (0 == strcmp(status,"disabled"))) + ban_pci_assigned_irq = 0; +} + void get_irq_data(struct irq_info *irq, void *data) { char **irqdata = (char **)data; @@ -619,6 +626,10 @@ int main(int argc, char** argv) foreground_mode=1; } + if (getenv("IRQBALANCE_AUTO_BAN_PCI_ASSIGNED_IRQS")) { + parse_ban_pci_assigned_irq(getenv("IRQBALANCE_AUTO_BAN_PCI_ASSIGNED_IRQS")); + } + /* * If we are't in debug mode, don't dump anything to the console * note that everything goes to the console before we check this diff --git a/irqbalance.h b/irqbalance.h index 5016cc8..339e2a3 100644 --- a/irqbalance.h +++ b/irqbalance.h @@ -103,8 +103,10 @@ extern void add_cl_banned_irq(int irq); #ifdef AARCH64 extern void add_banned_list_irq(int irq); #endif +extern void add_vm_banned_irq(int irq); extern void for_each_irq(GList *list, void (*cb)(struct irq_info *info, void *data), void *data); extern struct irq_info *get_irq_info(int irq); +extern int is_pci_assigned_irq(const char *line); extern void migrate_irq(GList **from, GList **to, struct irq_info *info); extern void free_cl_opts(void); extern void add_cl_banned_module(char *modname); diff --git a/procinterrupts.c b/procinterrupts.c index 6cfa661..522b9a1 100644 --- a/procinterrupts.c +++ b/procinterrupts.c @@ -42,6 +42,7 @@ static int proc_int_has_msi = 0; static int msi_found_in_sysfs = 0; +extern int ban_pci_assigned_irq; #ifdef AARCH64 struct irq_match { @@ -290,6 +291,12 @@ GList* collect_full_irq_list() c++; number = strtoul(line, NULL, 10); + if (ban_pci_assigned_irq && is_pci_assigned_irq(c)) { + log(TO_ALL, LOG_INFO, "Banned PCI-assigned irq %d.\n", number); + add_vm_banned_irq(number); + continue; + } + #ifdef AARCH64 if (is_arm_irq_aff_cannot_change(number)) { /* @@ -333,6 +340,7 @@ void parse_proc_interrupts(void) FILE *file; char *line = NULL; size_t size = 0; + int vfio_flag = 0; file = fopen("/proc/interrupts", "r"); if (!file) @@ -400,6 +408,21 @@ void parse_proc_interrupts(void) } info->existing = 1; + if (ban_pci_assigned_irq) { + vfio_flag = is_pci_assigned_irq(c); + if (vfio_flag && (info->flags & IRQ_FLAG_VM_BANNED) == 0) { + log(TO_ALL, LOG_INFO, "IRQ %d has been reused as a pci assigned interrupt number", number); + need_rescan = 1; + break; + } + + if (!vfio_flag && (info->flags & IRQ_FLAG_VM_BANNED) != 0) { + log(TO_ALL, LOG_INFO, "IRQ %d has been reused as a general interrupt number", number); + need_rescan = 1; + break; + } + } + count = 0; cpunr = 0; diff --git a/types.h b/types.h index 62cc2bb..c0950ee 100644 --- a/types.h +++ b/types.h @@ -35,6 +35,7 @@ * IRQ Internal tracking flags */ #define IRQ_FLAG_BANNED 1 +#define IRQ_FLAG_VM_BANNED 2 enum obj_type_e { OBJ_TYPE_CPU, -- 1.8.3.1