From 259bd39764e919721b27ecd6a21563b096788653 Mon Sep 17 00:00:00 2001 From: zouzhimin Date: Thu, 7 Mar 2024 01:05:44 +0000 Subject: [PATCH] !9 dlm_controld: update Linux kernel implementations From: @xiangbudaomz Reviewed-by: @jxy_git Signed-off-by: @jxy_git --- dlm.spec | 6 +- update-Linux-kernel-implementations.patch | 2687 +++++++++++++++++++++ 2 files changed, 2692 insertions(+), 1 deletion(-) create mode 100644 update-Linux-kernel-implementations.patch diff --git a/dlm.spec b/dlm.spec index 8421bfe..b5408aa 100644 --- a/dlm.spec +++ b/dlm.spec @@ -1,6 +1,6 @@ Name: dlm Version: 4.2.0 -Release: 3 +Release: 4 License: GPLv2 and GPLv2+ and LGPLv2+ Group: System Environment/Kernel Summary: dlm control daemon and tool @@ -15,6 +15,7 @@ BuildRequires: annobin Source0: https://releases.pagure.org/dlm/%{name}-%{version}.tar.gz Patch0001: 0030-dlm_controld-remove-unnecessary-header-include.patch Patch0002: fix-various-deadcode-issues.patch +Patch0003: update-Linux-kernel-implementations.patch Requires: %{name}-lib = %{version}-%{release} Requires: corosync >= 1.99.9 @@ -104,6 +105,9 @@ install -Dm 0644 init/dlm.sysconfig %{buildroot}/etc/sysconfig/dlm %{_libdir}/pkgconfig/*.pc %changelog +* Tue Mar 05 2024 zouzhimin - 4.2.0-4 +- dlm_controld: update Linux kernel implementations + * Mon Mar 04 2024 laokz - 4.2.0-3 - riscv64: remove -fcf-protection diff --git a/update-Linux-kernel-implementations.patch b/update-Linux-kernel-implementations.patch new file mode 100644 index 0000000..7928cd2 --- /dev/null +++ b/update-Linux-kernel-implementations.patch @@ -0,0 +1,2687 @@ +From abc2d9039fda5d57ed9bc0ec05b0f6cbc523bd99 Mon Sep 17 00:00:00 2001 +From: Alexander Aring +Date: Aug 21 2023 16:19:46 +0000 +Subject: [PATCH 1/3] dlm_controld: update rbtree implementation + + +This patch updates the rbtree implementation taken from the Linux +kernel. + +--- + +diff --git a/README.license b/README.license +index ededf66..d53f589 100644 +--- a/README.license ++++ b/README.license +@@ -7,10 +7,13 @@ libdlm/libdlm_internal.h + GPLv2 + + dlm_controld/list.h (copied from linux kernel) ++dlm_controld/linux_helpers.h (various GPLv2 functions copied from linux kernel) + + GPLv2+ + + dlm_controld/rbtree.c (copied from linux kernel) + dlm_controld/rbtree.h (copied from linux kernel) ++dlm_controld/rbtree_augmented.h (copied from linux kernel) ++dlm_controld/rbtree_types.h (copied from linux kernel) + all other original files + +diff --git a/dlm_controld/linux_helpers.h b/dlm_controld/linux_helpers.h +new file mode 100644 +index 0000000..5ef1346 +--- /dev/null ++++ b/dlm_controld/linux_helpers.h +@@ -0,0 +1,11 @@ ++/* Copied from linux kernel */ ++ ++#ifndef __DLM_LINUX_HELPERS__ ++#define __DLM_LINUX_HELPERS__ ++ ++#define WRITE_ONCE(x, val) \ ++do { \ ++ *(volatile typeof(x) *)&(x) = (val); \ ++} while (0) ++ ++#endif /* __DLM_LINUX_HELPERS__ */ +diff --git a/dlm_controld/list.h b/dlm_controld/list.h +index 8100cbc..a2a5e5f 100644 +--- a/dlm_controld/list.h ++++ b/dlm_controld/list.h +@@ -15,7 +15,6 @@ + const typeof( ((type *)0)->member ) *__mptr = (ptr); \ + (type *)( (char *)__mptr - offsetof(type,member) );}) + +- + /* + * These are non-NULL pointers that will result in page faults + * under normal circumstances, used to verify that nobody uses +diff --git a/dlm_controld/rbtree.c b/dlm_controld/rbtree.c +index 9f49917..d07ffc2 100644 +--- a/dlm_controld/rbtree.c ++++ b/dlm_controld/rbtree.c +@@ -1,284 +1,458 @@ ++/* Copied from linux/lib/rbtree.c */ + /* + Red Black Trees + (C) 1999 Andrea Arcangeli + (C) 2002 David Woodhouse +- +- This program is free software; you can redistribute it and/or modify +- it under the terms of the GNU General Public License as published by +- the Free Software Foundation; either version 2 of the License, or +- (at your option) any later version. ++ (C) 2012 Michel Lespinasse + +- This program is distributed in the hope that it will be useful, +- but WITHOUT ANY WARRANTY; without even the implied warranty of +- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +- GNU General Public License for more details. +- +- You should have received a copy of the GNU General Public License +- along with this program; if not, write to the Free Software +- Foundation, Inc. + + linux/lib/rbtree.c + */ + +-#include +-#include "rbtree.h" +- +-static void __rb_rotate_left(struct rb_node *node, struct rb_root *root) +-{ +- struct rb_node *right = node->rb_right; +- struct rb_node *parent = rb_parent(node); ++#include "rbtree_augmented.h" + +- if ((node->rb_right = right->rb_left)) +- rb_set_parent(right->rb_left, node); +- right->rb_left = node; ++/* ++ * red-black trees properties: https://en.wikipedia.org/wiki/Rbtree ++ * ++ * 1) A node is either red or black ++ * 2) The root is black ++ * 3) All leaves (NULL) are black ++ * 4) Both children of every red node are black ++ * 5) Every simple path from root to leaves contains the same number ++ * of black nodes. ++ * ++ * 4 and 5 give the O(log n) guarantee, since 4 implies you cannot have two ++ * consecutive red nodes in a path and every red node is therefore followed by ++ * a black. So if B is the number of black nodes on every simple path (as per ++ * 5), then the longest possible path due to 4 is 2B. ++ * ++ * We shall indicate color with case, where black nodes are uppercase and red ++ * nodes will be lowercase. Unknown color nodes shall be drawn as red within ++ * parentheses and have some accompanying text comment. ++ */ + +- rb_set_parent(right, parent); ++/* ++ * Notes on lockless lookups: ++ * ++ * All stores to the tree structure (rb_left and rb_right) must be done using ++ * WRITE_ONCE(). And we must not inadvertently cause (temporary) loops in the ++ * tree structure as seen in program order. ++ * ++ * These two requirements will allow lockless iteration of the tree -- not ++ * correct iteration mind you, tree rotations are not atomic so a lookup might ++ * miss entire subtrees. ++ * ++ * But they do guarantee that any such traversal will only see valid elements ++ * and that it will indeed complete -- does not get stuck in a loop. ++ * ++ * It also guarantees that if the lookup returns an element it is the 'correct' ++ * one. But not returning an element does _NOT_ mean it's not present. ++ * ++ * NOTE: ++ * ++ * Stores to __rb_parent_color are not important for simple lookups so those ++ * are left undone as of now. Nor did I check for loops involving parent ++ * pointers. ++ */ + +- if (parent) +- { +- if (node == parent->rb_left) +- parent->rb_left = right; +- else +- parent->rb_right = right; +- } +- else +- root->rb_node = right; +- rb_set_parent(node, right); ++static inline void rb_set_black(struct rb_node *rb) ++{ ++ rb->__rb_parent_color += RB_BLACK; + } + +-static void __rb_rotate_right(struct rb_node *node, struct rb_root *root) ++static inline struct rb_node *rb_red_parent(struct rb_node *red) + { +- struct rb_node *left = node->rb_left; +- struct rb_node *parent = rb_parent(node); +- +- if ((node->rb_left = left->rb_right)) +- rb_set_parent(left->rb_right, node); +- left->rb_right = node; +- +- rb_set_parent(left, parent); ++ return (struct rb_node *)red->__rb_parent_color; ++} + +- if (parent) +- { +- if (node == parent->rb_right) +- parent->rb_right = left; +- else +- parent->rb_left = left; +- } +- else +- root->rb_node = left; +- rb_set_parent(node, left); ++/* ++ * Helper function for rotations: ++ * - old's parent and color get assigned to new ++ * - old gets assigned new as a parent and 'color' as a color. ++ */ ++static inline void ++__rb_rotate_set_parents(struct rb_node *old, struct rb_node *new, ++ struct rb_root *root, int color) ++{ ++ struct rb_node *parent = rb_parent(old); ++ new->__rb_parent_color = old->__rb_parent_color; ++ rb_set_parent_color(old, new, color); ++ __rb_change_child(old, new, parent, root); + } + +-void rb_insert_color(struct rb_node *node, struct rb_root *root) ++static __always_inline void ++__rb_insert(struct rb_node *node, struct rb_root *root, ++ void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) + { +- struct rb_node *parent, *gparent; +- +- while ((parent = rb_parent(node)) && rb_is_red(parent)) +- { +- gparent = rb_parent(parent); +- +- if (parent == gparent->rb_left) +- { +- { +- register struct rb_node *uncle = gparent->rb_right; +- if (uncle && rb_is_red(uncle)) +- { +- rb_set_black(uncle); +- rb_set_black(parent); +- rb_set_red(gparent); +- node = gparent; +- continue; +- } ++ struct rb_node *parent = rb_red_parent(node), *gparent, *tmp; ++ ++ while (true) { ++ /* ++ * Loop invariant: node is red. ++ */ ++ if (!parent) { ++ /* ++ * The inserted node is root. Either this is the ++ * first node, or we recursed at Case 1 below and ++ * are no longer violating 4). ++ */ ++ rb_set_parent_color(node, NULL, RB_BLACK); ++ break; ++ } ++ ++ /* ++ * If there is a black parent, we are done. ++ * Otherwise, take some corrective action as, ++ * per 4), we don't want a red root or two ++ * consecutive red nodes. ++ */ ++ if(rb_is_black(parent)) ++ break; ++ ++ gparent = rb_red_parent(parent); ++ ++ tmp = gparent->rb_right; ++ if (parent != tmp) { /* parent == gparent->rb_left */ ++ if (tmp && rb_is_red(tmp)) { ++ /* ++ * Case 1 - node's uncle is red (color flips). ++ * ++ * G g ++ * / \ / \ ++ * p u --> P U ++ * / / ++ * n n ++ * ++ * However, since g's parent might be red, and ++ * 4) does not allow this, we need to recurse ++ * at g. ++ */ ++ rb_set_parent_color(tmp, gparent, RB_BLACK); ++ rb_set_parent_color(parent, gparent, RB_BLACK); ++ node = gparent; ++ parent = rb_parent(node); ++ rb_set_parent_color(node, parent, RB_RED); ++ continue; + } + +- if (parent->rb_right == node) +- { +- register struct rb_node *tmp; +- __rb_rotate_left(parent, root); +- tmp = parent; ++ tmp = parent->rb_right; ++ if (node == tmp) { ++ /* ++ * Case 2 - node's uncle is black and node is ++ * the parent's right child (left rotate at parent). ++ * ++ * G G ++ * / \ / \ ++ * p U --> n U ++ * \ / ++ * n p ++ * ++ * This still leaves us in violation of 4), the ++ * continuation into Case 3 will fix that. ++ */ ++ tmp = node->rb_left; ++ WRITE_ONCE(parent->rb_right, tmp); ++ WRITE_ONCE(node->rb_left, parent); ++ if (tmp) ++ rb_set_parent_color(tmp, parent, ++ RB_BLACK); ++ rb_set_parent_color(parent, node, RB_RED); ++ augment_rotate(parent, node); + parent = node; +- node = tmp; ++ tmp = node->rb_right; + } + +- rb_set_black(parent); +- rb_set_red(gparent); +- __rb_rotate_right(gparent, root); ++ /* ++ * Case 3 - node's uncle is black and node is ++ * the parent's left child (right rotate at gparent). ++ * ++ * G P ++ * / \ / \ ++ * p U --> n g ++ * / \ ++ * n U ++ */ ++ WRITE_ONCE(gparent->rb_left, tmp); /* == parent->rb_right */ ++ WRITE_ONCE(parent->rb_right, gparent); ++ if (tmp) ++ rb_set_parent_color(tmp, gparent, RB_BLACK); ++ __rb_rotate_set_parents(gparent, parent, root, RB_RED); ++ augment_rotate(gparent, parent); ++ break; + } else { +- { +- register struct rb_node *uncle = gparent->rb_left; +- if (uncle && rb_is_red(uncle)) +- { +- rb_set_black(uncle); +- rb_set_black(parent); +- rb_set_red(gparent); +- node = gparent; +- continue; +- } ++ tmp = gparent->rb_left; ++ if (tmp && rb_is_red(tmp)) { ++ /* Case 1 - color flips */ ++ rb_set_parent_color(tmp, gparent, RB_BLACK); ++ rb_set_parent_color(parent, gparent, RB_BLACK); ++ node = gparent; ++ parent = rb_parent(node); ++ rb_set_parent_color(node, parent, RB_RED); ++ continue; + } + +- if (parent->rb_left == node) +- { +- register struct rb_node *tmp; +- __rb_rotate_right(parent, root); +- tmp = parent; ++ tmp = parent->rb_left; ++ if (node == tmp) { ++ /* Case 2 - right rotate at parent */ ++ tmp = node->rb_right; ++ WRITE_ONCE(parent->rb_left, tmp); ++ WRITE_ONCE(node->rb_right, parent); ++ if (tmp) ++ rb_set_parent_color(tmp, parent, ++ RB_BLACK); ++ rb_set_parent_color(parent, node, RB_RED); ++ augment_rotate(parent, node); + parent = node; +- node = tmp; ++ tmp = node->rb_left; + } + +- rb_set_black(parent); +- rb_set_red(gparent); +- __rb_rotate_left(gparent, root); ++ /* Case 3 - left rotate at gparent */ ++ WRITE_ONCE(gparent->rb_right, tmp); /* == parent->rb_left */ ++ WRITE_ONCE(parent->rb_left, gparent); ++ if (tmp) ++ rb_set_parent_color(tmp, gparent, RB_BLACK); ++ __rb_rotate_set_parents(gparent, parent, root, RB_RED); ++ augment_rotate(gparent, parent); ++ break; + } + } +- +- rb_set_black(root->rb_node); + } + +-static void __rb_erase_color(struct rb_node *node, struct rb_node *parent, +- struct rb_root *root) ++/* ++ * Inline version for rb_erase() use - we want to be able to inline ++ * and eliminate the dummy_rotate callback there ++ */ ++static __always_inline void ++____rb_erase_color(struct rb_node *parent, struct rb_root *root, ++ void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) + { +- struct rb_node *other; +- +- while ((!node || rb_is_black(node)) && node != root->rb_node) +- { +- if (parent->rb_left == node) +- { +- other = parent->rb_right; +- if (rb_is_red(other)) +- { +- rb_set_black(other); +- rb_set_red(parent); +- __rb_rotate_left(parent, root); +- other = parent->rb_right; +- } +- if ((!other->rb_left || rb_is_black(other->rb_left)) && +- (!other->rb_right || rb_is_black(other->rb_right))) +- { +- rb_set_red(other); +- node = parent; +- parent = rb_parent(node); ++ struct rb_node *node = NULL, *sibling, *tmp1, *tmp2; ++ ++ while (true) { ++ /* ++ * Loop invariants: ++ * - node is black (or NULL on first iteration) ++ * - node is not the root (parent is not NULL) ++ * - All leaf paths going through parent and node have a ++ * black node count that is 1 lower than other leaf paths. ++ */ ++ sibling = parent->rb_right; ++ if (node != sibling) { /* node == parent->rb_left */ ++ if (rb_is_red(sibling)) { ++ /* ++ * Case 1 - left rotate at parent ++ * ++ * P S ++ * / \ / \ ++ * N s --> p Sr ++ * / \ / \ ++ * Sl Sr N Sl ++ */ ++ tmp1 = sibling->rb_left; ++ WRITE_ONCE(parent->rb_right, tmp1); ++ WRITE_ONCE(sibling->rb_left, parent); ++ rb_set_parent_color(tmp1, parent, RB_BLACK); ++ __rb_rotate_set_parents(parent, sibling, root, ++ RB_RED); ++ augment_rotate(parent, sibling); ++ sibling = tmp1; + } +- else +- { +- if (!other->rb_right || rb_is_black(other->rb_right)) +- { +- rb_set_black(other->rb_left); +- rb_set_red(other); +- __rb_rotate_right(other, root); +- other = parent->rb_right; ++ tmp1 = sibling->rb_right; ++ if (!tmp1 || rb_is_black(tmp1)) { ++ tmp2 = sibling->rb_left; ++ if (!tmp2 || rb_is_black(tmp2)) { ++ /* ++ * Case 2 - sibling color flip ++ * (p could be either color here) ++ * ++ * (p) (p) ++ * / \ / \ ++ * N S --> N s ++ * / \ / \ ++ * Sl Sr Sl Sr ++ * ++ * This leaves us violating 5) which ++ * can be fixed by flipping p to black ++ * if it was red, or by recursing at p. ++ * p is red when coming from Case 1. ++ */ ++ rb_set_parent_color(sibling, parent, ++ RB_RED); ++ if (rb_is_red(parent)) ++ rb_set_black(parent); ++ else { ++ node = parent; ++ parent = rb_parent(node); ++ if (parent) ++ continue; ++ } ++ break; + } +- rb_set_color(other, rb_color(parent)); +- rb_set_black(parent); +- rb_set_black(other->rb_right); +- __rb_rotate_left(parent, root); +- node = root->rb_node; +- break; ++ /* ++ * Case 3 - right rotate at sibling ++ * (p could be either color here) ++ * ++ * (p) (p) ++ * / \ / \ ++ * N S --> N sl ++ * / \ \ ++ * sl Sr S ++ * \ ++ * Sr ++ * ++ * Note: p might be red, and then both ++ * p and sl are red after rotation(which ++ * breaks property 4). This is fixed in ++ * Case 4 (in __rb_rotate_set_parents() ++ * which set sl the color of p ++ * and set p RB_BLACK) ++ * ++ * (p) (sl) ++ * / \ / \ ++ * N sl --> P S ++ * \ / \ ++ * S N Sr ++ * \ ++ * Sr ++ */ ++ tmp1 = tmp2->rb_right; ++ WRITE_ONCE(sibling->rb_left, tmp1); ++ WRITE_ONCE(tmp2->rb_right, sibling); ++ WRITE_ONCE(parent->rb_right, tmp2); ++ if (tmp1) ++ rb_set_parent_color(tmp1, sibling, ++ RB_BLACK); ++ augment_rotate(sibling, tmp2); ++ tmp1 = sibling; ++ sibling = tmp2; + } +- } +- else +- { +- other = parent->rb_left; +- if (rb_is_red(other)) +- { +- rb_set_black(other); +- rb_set_red(parent); +- __rb_rotate_right(parent, root); +- other = parent->rb_left; +- } +- if ((!other->rb_left || rb_is_black(other->rb_left)) && +- (!other->rb_right || rb_is_black(other->rb_right))) +- { +- rb_set_red(other); +- node = parent; +- parent = rb_parent(node); ++ /* ++ * Case 4 - left rotate at parent + color flips ++ * (p and sl could be either color here. ++ * After rotation, p becomes black, s acquires ++ * p's color, and sl keeps its color) ++ * ++ * (p) (s) ++ * / \ / \ ++ * N S --> P Sr ++ * / \ / \ ++ * (sl) sr N (sl) ++ */ ++ tmp2 = sibling->rb_left; ++ WRITE_ONCE(parent->rb_right, tmp2); ++ WRITE_ONCE(sibling->rb_left, parent); ++ rb_set_parent_color(tmp1, sibling, RB_BLACK); ++ if (tmp2) ++ rb_set_parent(tmp2, parent); ++ __rb_rotate_set_parents(parent, sibling, root, ++ RB_BLACK); ++ augment_rotate(parent, sibling); ++ break; ++ } else { ++ sibling = parent->rb_left; ++ if (rb_is_red(sibling)) { ++ /* Case 1 - right rotate at parent */ ++ tmp1 = sibling->rb_right; ++ WRITE_ONCE(parent->rb_left, tmp1); ++ WRITE_ONCE(sibling->rb_right, parent); ++ rb_set_parent_color(tmp1, parent, RB_BLACK); ++ __rb_rotate_set_parents(parent, sibling, root, ++ RB_RED); ++ augment_rotate(parent, sibling); ++ sibling = tmp1; + } +- else +- { +- if (!other->rb_left || rb_is_black(other->rb_left)) +- { +- rb_set_black(other->rb_right); +- rb_set_red(other); +- __rb_rotate_left(other, root); +- other = parent->rb_left; ++ tmp1 = sibling->rb_left; ++ if (!tmp1 || rb_is_black(tmp1)) { ++ tmp2 = sibling->rb_right; ++ if (!tmp2 || rb_is_black(tmp2)) { ++ /* Case 2 - sibling color flip */ ++ rb_set_parent_color(sibling, parent, ++ RB_RED); ++ if (rb_is_red(parent)) ++ rb_set_black(parent); ++ else { ++ node = parent; ++ parent = rb_parent(node); ++ if (parent) ++ continue; ++ } ++ break; + } +- rb_set_color(other, rb_color(parent)); +- rb_set_black(parent); +- rb_set_black(other->rb_left); +- __rb_rotate_right(parent, root); +- node = root->rb_node; +- break; ++ /* Case 3 - left rotate at sibling */ ++ tmp1 = tmp2->rb_left; ++ WRITE_ONCE(sibling->rb_right, tmp1); ++ WRITE_ONCE(tmp2->rb_left, sibling); ++ WRITE_ONCE(parent->rb_left, tmp2); ++ if (tmp1) ++ rb_set_parent_color(tmp1, sibling, ++ RB_BLACK); ++ augment_rotate(sibling, tmp2); ++ tmp1 = sibling; ++ sibling = tmp2; + } ++ /* Case 4 - right rotate at parent + color flips */ ++ tmp2 = sibling->rb_right; ++ WRITE_ONCE(parent->rb_left, tmp2); ++ WRITE_ONCE(sibling->rb_right, parent); ++ rb_set_parent_color(tmp1, sibling, RB_BLACK); ++ if (tmp2) ++ rb_set_parent(tmp2, parent); ++ __rb_rotate_set_parents(parent, sibling, root, ++ RB_BLACK); ++ augment_rotate(parent, sibling); ++ break; + } + } +- if (node) +- rb_set_black(node); + } + +-void rb_erase(struct rb_node *node, struct rb_root *root) ++/* Non-inline version for rb_erase_augmented() use */ ++void __rb_erase_color(struct rb_node *parent, struct rb_root *root, ++ void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) + { +- struct rb_node *child, *parent; +- int color; ++ ____rb_erase_color(parent, root, augment_rotate); ++} + +- if (!node->rb_left) +- child = node->rb_right; +- else if (!node->rb_right) +- child = node->rb_left; +- else +- { +- struct rb_node *old = node, *left; ++/* ++ * Non-augmented rbtree manipulation functions. ++ * ++ * We use dummy augmented callbacks here, and have the compiler optimize them ++ * out of the rb_insert_color() and rb_erase() function definitions. ++ */ + +- node = node->rb_right; +- while ((left = node->rb_left) != NULL) +- node = left; +- +- if (rb_parent(old)) { +- if (rb_parent(old)->rb_left == old) +- rb_parent(old)->rb_left = node; +- else +- rb_parent(old)->rb_right = node; +- } else +- root->rb_node = node; +- +- child = node->rb_right; +- parent = rb_parent(node); +- color = rb_color(node); +- +- if (parent == old) { +- parent = node; +- } else { +- if (child) +- rb_set_parent(child, parent); +- parent->rb_left = child; ++static inline void dummy_propagate(struct rb_node *node, struct rb_node *stop) {} ++static inline void dummy_copy(struct rb_node *old, struct rb_node *new) {} ++static inline void dummy_rotate(struct rb_node *old, struct rb_node *new) {} + +- node->rb_right = old->rb_right; +- rb_set_parent(old->rb_right, node); +- } ++static const struct rb_augment_callbacks dummy_callbacks = { ++ .propagate = dummy_propagate, ++ .copy = dummy_copy, ++ .rotate = dummy_rotate ++}; + +- node->rb_parent_color = old->rb_parent_color; +- node->rb_left = old->rb_left; +- rb_set_parent(old->rb_left, node); ++void rb_insert_color(struct rb_node *node, struct rb_root *root) ++{ ++ __rb_insert(node, root, dummy_rotate); ++} + +- goto color; +- } ++void rb_erase(struct rb_node *node, struct rb_root *root) ++{ ++ struct rb_node *rebalance; ++ rebalance = __rb_erase_augmented(node, root, &dummy_callbacks); ++ if (rebalance) ++ ____rb_erase_color(rebalance, root, dummy_rotate); ++} + +- parent = rb_parent(node); +- color = rb_color(node); +- +- if (child) +- rb_set_parent(child, parent); +- if (parent) +- { +- if (parent->rb_left == node) +- parent->rb_left = child; +- else +- parent->rb_right = child; +- } +- else +- root->rb_node = child; ++/* ++ * Augmented rbtree manipulation functions. ++ * ++ * This instantiates the same __always_inline functions as in the non-augmented ++ * case, but this time with user-defined callbacks. ++ */ + +- color: +- if (color == RB_BLACK) +- __rb_erase_color(child, parent, root); ++void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, ++ void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) ++{ ++ __rb_insert(node, root, augment_rotate); + } + + /* +@@ -312,24 +486,27 @@ struct rb_node *rb_next(const struct rb_node *node) + { + struct rb_node *parent; + +- if (rb_parent(node) == node) ++ if (RB_EMPTY_NODE(node)) + return NULL; + +- /* If we have a right-hand child, go down and then left as far +- as we can. */ ++ /* ++ * If we have a right-hand child, go down and then left as far ++ * as we can. ++ */ + if (node->rb_right) { +- node = node->rb_right; ++ node = node->rb_right; + while (node->rb_left) +- node=node->rb_left; ++ node = node->rb_left; + return (struct rb_node *)node; + } + +- /* No right-hand children. Everything down and left is +- smaller than us, so any 'next' node must be in the general +- direction of our parent. Go up the tree; any time the +- ancestor is a right-hand child of its parent, keep going +- up. First time it's a left-hand child of its parent, said +- parent is our 'next' node. */ ++ /* ++ * No right-hand children. Everything down and left is smaller than us, ++ * so any 'next' node must be in the general direction of our parent. ++ * Go up the tree; any time the ancestor is a right-hand child of its ++ * parent, keep going up. First time it's a left-hand child of its ++ * parent, said parent is our 'next' node. ++ */ + while ((parent = rb_parent(node)) && node == parent->rb_right) + node = parent; + +@@ -340,20 +517,24 @@ struct rb_node *rb_prev(const struct rb_node *node) + { + struct rb_node *parent; + +- if (rb_parent(node) == node) ++ if (RB_EMPTY_NODE(node)) + return NULL; + +- /* If we have a left-hand child, go down and then right as far +- as we can. */ ++ /* ++ * If we have a left-hand child, go down and then right as far ++ * as we can. ++ */ + if (node->rb_left) { +- node = node->rb_left; ++ node = node->rb_left; + while (node->rb_right) +- node=node->rb_right; ++ node = node->rb_right; + return (struct rb_node *)node; + } + +- /* No left-hand children. Go up till we find an ancestor which +- is a right-hand child of its parent */ ++ /* ++ * No left-hand children. Go up till we find an ancestor which ++ * is a right-hand child of its parent. ++ */ + while ((parent = rb_parent(node)) && node == parent->rb_left) + node = parent; + +@@ -365,20 +546,51 @@ void rb_replace_node(struct rb_node *victim, struct rb_node *new, + { + struct rb_node *parent = rb_parent(victim); + ++ /* Copy the pointers/colour from the victim to the replacement */ ++ *new = *victim; ++ + /* Set the surrounding nodes to point to the replacement */ +- if (parent) { +- if (victim == parent->rb_left) +- parent->rb_left = new; +- else +- parent->rb_right = new; +- } else { +- root->rb_node = new; +- } + if (victim->rb_left) + rb_set_parent(victim->rb_left, new); + if (victim->rb_right) + rb_set_parent(victim->rb_right, new); ++ __rb_change_child(victim, new, parent, root); ++} + +- /* Copy the pointers/colour from the victim to the replacement */ +- *new = *victim; ++static struct rb_node *rb_left_deepest_node(const struct rb_node *node) ++{ ++ for (;;) { ++ if (node->rb_left) ++ node = node->rb_left; ++ else if (node->rb_right) ++ node = node->rb_right; ++ else ++ return (struct rb_node *)node; ++ } ++} ++ ++struct rb_node *rb_next_postorder(const struct rb_node *node) ++{ ++ const struct rb_node *parent; ++ if (!node) ++ return NULL; ++ parent = rb_parent(node); ++ ++ /* If we're sitting on node, we've already seen our children */ ++ if (parent && node == parent->rb_left && parent->rb_right) { ++ /* If we are the parent's left node, go to the parent's right ++ * node then all the way down to the left */ ++ return rb_left_deepest_node(parent->rb_right); ++ } else ++ /* Otherwise we are the parent's right node, and the parent ++ * should be next */ ++ return (struct rb_node *)parent; ++} ++ ++struct rb_node *rb_first_postorder(const struct rb_root *root) ++{ ++ if (!root->rb_node) ++ return NULL; ++ ++ return rb_left_deepest_node(root->rb_node); + } +diff --git a/dlm_controld/rbtree.h b/dlm_controld/rbtree.h +index 9a4ad60..ddb86ff 100644 +--- a/dlm_controld/rbtree.h ++++ b/dlm_controld/rbtree.h +@@ -1,20 +1,8 @@ ++/* Copied from linux/include/linux/rbtree.h */ + /* + Red Black Trees + (C) 1999 Andrea Arcangeli + +- This program is free software; you can redistribute it and/or modify +- it under the terms of the GNU General Public License as published by +- the Free Software Foundation; either version 2 of the License, or +- (at your option) any later version. +- +- This program is distributed in the hope that it will be useful, +- but WITHOUT ANY WARRANTY; without even the implied warranty of +- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +- GNU General Public License for more details. +- +- You should have received a copy of the GNU General Public License +- along with this program; if not, write to the Free Software +- Foundation, Inc. + + linux/include/linux/rbtree.h + +@@ -23,138 +11,313 @@ + I know it's not the cleaner way, but in C (not in C++) to get + performances and genericity... + +- Some example of insert and search follows here. The search is a plain +- normal search over an ordered tree. The insert instead must be implemented +- int two steps: as first thing the code must insert the element in +- order as a red leaf in the tree, then the support library function +- rb_insert_color() must be called. Such function will do the +- not trivial work to rebalance the rbtree if necessary. ++ See Documentation/core-api/rbtree.rst for documentation and samples. ++*/ ++ ++#ifndef _LINUX_RBTREE_H ++#define _LINUX_RBTREE_H ++ ++#include ++#include ++ ++#include "list.h" ++#include "rbtree_types.h" ++ ++#define rb_parent(r) ((struct rb_node *)((r)->__rb_parent_color & ~3)) ++ ++#define rb_entry(ptr, type, member) container_of(ptr, type, member) ++ ++#define RB_EMPTY_ROOT(root) (READ_ONCE((root)->rb_node) == NULL) ++ ++/* 'empty' nodes are nodes that are known not to be inserted in an rbtree */ ++#define RB_EMPTY_NODE(node) \ ++ ((node)->__rb_parent_color == (unsigned long)(node)) ++#define RB_CLEAR_NODE(node) \ ++ ((node)->__rb_parent_color = (unsigned long)(node)) ++ ++ ++extern void rb_insert_color(struct rb_node *, struct rb_root *); ++extern void rb_erase(struct rb_node *, struct rb_root *); ++ ++ ++/* Find logical next and previous nodes in a tree */ ++extern struct rb_node *rb_next(const struct rb_node *); ++extern struct rb_node *rb_prev(const struct rb_node *); ++extern struct rb_node *rb_first(const struct rb_root *); ++extern struct rb_node *rb_last(const struct rb_root *); ++ ++/* Postorder iteration - always visit the parent after its children */ ++extern struct rb_node *rb_first_postorder(const struct rb_root *); ++extern struct rb_node *rb_next_postorder(const struct rb_node *); ++ ++/* Fast replacement of a single node without remove/rebalance/add/rebalance */ ++extern void rb_replace_node(struct rb_node *victim, struct rb_node *new, ++ struct rb_root *root); + +------------------------------------------------------------------------ +-static inline struct page * rb_search_page_cache(struct inode * inode, +- unsigned long offset) ++static inline void rb_link_node(struct rb_node *node, struct rb_node *parent, ++ struct rb_node **rb_link) + { +- struct rb_node * n = inode->i_rb_page_cache.rb_node; +- struct page * page; ++ node->__rb_parent_color = (unsigned long)parent; ++ node->rb_left = node->rb_right = NULL; + +- while (n) +- { +- page = rb_entry(n, struct page, rb_page_cache); ++ *rb_link = node; ++} + +- if (offset < page->offset) +- n = n->rb_left; +- else if (offset > page->offset) +- n = n->rb_right; +- else +- return page; +- } +- return NULL; ++#define rb_entry_safe(ptr, type, member) \ ++ ({ typeof(ptr) ____ptr = (ptr); \ ++ ____ptr ? rb_entry(____ptr, type, member) : NULL; \ ++ }) ++ ++/** ++ * rbtree_postorder_for_each_entry_safe - iterate in post-order over rb_root of ++ * given type allowing the backing memory of @pos to be invalidated ++ * ++ * @pos: the 'type *' to use as a loop cursor. ++ * @n: another 'type *' to use as temporary storage ++ * @root: 'rb_root *' of the rbtree. ++ * @field: the name of the rb_node field within 'type'. ++ * ++ * rbtree_postorder_for_each_entry_safe() provides a similar guarantee as ++ * list_for_each_entry_safe() and allows the iteration to continue independent ++ * of changes to @pos by the body of the loop. ++ * ++ * Note, however, that it cannot handle other modifications that re-order the ++ * rbtree it is iterating over. This includes calling rb_erase() on @pos, as ++ * rb_erase() may rebalance the tree, causing us to miss some nodes. ++ */ ++#define rbtree_postorder_for_each_entry_safe(pos, n, root, field) \ ++ for (pos = rb_entry_safe(rb_first_postorder(root), typeof(*pos), field); \ ++ pos && ({ n = rb_entry_safe(rb_next_postorder(&pos->field), \ ++ typeof(*pos), field); 1; }); \ ++ pos = n) ++ ++/* Same as rb_first(), but O(1) */ ++#define rb_first_cached(root) (root)->rb_leftmost ++ ++static inline void rb_insert_color_cached(struct rb_node *node, ++ struct rb_root_cached *root, ++ bool leftmost) ++{ ++ if (leftmost) ++ root->rb_leftmost = node; ++ rb_insert_color(node, &root->rb_root); + } + +-static inline struct page * __rb_insert_page_cache(struct inode * inode, +- unsigned long offset, +- struct rb_node * node) ++ ++static inline struct rb_node * ++rb_erase_cached(struct rb_node *node, struct rb_root_cached *root) + { +- struct rb_node ** p = &inode->i_rb_page_cache.rb_node; +- struct rb_node * parent = NULL; +- struct page * page; +- +- while (*p) +- { +- parent = *p; +- page = rb_entry(parent, struct page, rb_page_cache); +- +- if (offset < page->offset) +- p = &(*p)->rb_left; +- else if (offset > page->offset) +- p = &(*p)->rb_right; +- else +- return page; +- } ++ struct rb_node *leftmost = NULL; + +- rb_link_node(node, parent, p); ++ if (root->rb_leftmost == node) ++ leftmost = root->rb_leftmost = rb_next(node); + +- return NULL; ++ rb_erase(node, &root->rb_root); ++ ++ return leftmost; + } + +-static inline struct page * rb_insert_page_cache(struct inode * inode, +- unsigned long offset, +- struct rb_node * node) ++static inline void rb_replace_node_cached(struct rb_node *victim, ++ struct rb_node *new, ++ struct rb_root_cached *root) + { +- struct page * ret; +- if ((ret = __rb_insert_page_cache(inode, offset, node))) +- goto out; +- rb_insert_color(node, &inode->i_rb_page_cache); +- out: +- return ret; ++ if (root->rb_leftmost == victim) ++ root->rb_leftmost = new; ++ rb_replace_node(victim, new, &root->rb_root); + } +------------------------------------------------------------------------ +-*/ + +-#ifndef _LINUX_RBTREE_H +-#define _LINUX_RBTREE_H +- +-#include ++/* ++ * The below helper functions use 2 operators with 3 different ++ * calling conventions. The operators are related like: ++ * ++ * comp(a->key,b) < 0 := less(a,b) ++ * comp(a->key,b) > 0 := less(b,a) ++ * comp(a->key,b) == 0 := !less(a,b) && !less(b,a) ++ * ++ * If these operators define a partial order on the elements we make no ++ * guarantee on which of the elements matching the key is found. See ++ * rb_find(). ++ * ++ * The reason for this is to allow the find() interface without requiring an ++ * on-stack dummy object, which might not be feasible due to object size. ++ */ + +-struct rb_node +-{ +- unsigned long rb_parent_color; +-#define RB_RED 0 +-#define RB_BLACK 1 +- struct rb_node *rb_right; +- struct rb_node *rb_left; +-} __attribute__((aligned(sizeof(long)))); +- /* The alignment might seem pointless, but allegedly CRIS needs it */ +- +-struct rb_root ++/** ++ * rb_add_cached() - insert @node into the leftmost cached tree @tree ++ * @node: node to insert ++ * @tree: leftmost cached tree to insert @node into ++ * @less: operator defining the (partial) node order ++ * ++ * Returns @node when it is the new leftmost, or NULL. ++ */ ++static __always_inline struct rb_node * ++rb_add_cached(struct rb_node *node, struct rb_root_cached *tree, ++ bool (*less)(struct rb_node *, const struct rb_node *)) + { +- struct rb_node *rb_node; +-}; ++ struct rb_node **link = &tree->rb_root.rb_node; ++ struct rb_node *parent = NULL; ++ bool leftmost = true; + ++ while (*link) { ++ parent = *link; ++ if (less(node, parent)) { ++ link = &parent->rb_left; ++ } else { ++ link = &parent->rb_right; ++ leftmost = false; ++ } ++ } + +-#define rb_parent(r) ((struct rb_node *)((r)->rb_parent_color & ~3)) +-#define rb_color(r) ((r)->rb_parent_color & 1) +-#define rb_is_red(r) (!rb_color(r)) +-#define rb_is_black(r) rb_color(r) +-#define rb_set_red(r) do { (r)->rb_parent_color &= ~1; } while (0) +-#define rb_set_black(r) do { (r)->rb_parent_color |= 1; } while (0) ++ rb_link_node(node, parent, link); ++ rb_insert_color_cached(node, tree, leftmost); + +-static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p) +-{ +- rb->rb_parent_color = (rb->rb_parent_color & 3) | (unsigned long)p; ++ return leftmost ? node : NULL; + } +-static inline void rb_set_color(struct rb_node *rb, int color) ++ ++/** ++ * rb_add() - insert @node into @tree ++ * @node: node to insert ++ * @tree: tree to insert @node into ++ * @less: operator defining the (partial) node order ++ */ ++static __always_inline void ++rb_add(struct rb_node *node, struct rb_root *tree, ++ bool (*less)(struct rb_node *, const struct rb_node *)) + { +- rb->rb_parent_color = (rb->rb_parent_color & ~1) | color; ++ struct rb_node **link = &tree->rb_node; ++ struct rb_node *parent = NULL; ++ ++ while (*link) { ++ parent = *link; ++ if (less(node, parent)) ++ link = &parent->rb_left; ++ else ++ link = &parent->rb_right; ++ } ++ ++ rb_link_node(node, parent, link); ++ rb_insert_color(node, tree); + } + +-#define RB_ROOT (struct rb_root) { NULL, } +-#define rb_entry(ptr, type, member) container_of(ptr, type, member) ++/** ++ * rb_find_add() - find equivalent @node in @tree, or add @node ++ * @node: node to look-for / insert ++ * @tree: tree to search / modify ++ * @cmp: operator defining the node order ++ * ++ * Returns the rb_node matching @node, or NULL when no match is found and @node ++ * is inserted. ++ */ ++static __always_inline struct rb_node * ++rb_find_add(struct rb_node *node, struct rb_root *tree, ++ int (*cmp)(struct rb_node *, const struct rb_node *)) ++{ ++ struct rb_node **link = &tree->rb_node; ++ struct rb_node *parent = NULL; ++ int c; + +-#define RB_EMPTY_ROOT(root) ((root)->rb_node == NULL) +-#define RB_EMPTY_NODE(node) (rb_parent(node) == node) +-#define RB_CLEAR_NODE(node) (rb_set_parent(node, node)) ++ while (*link) { ++ parent = *link; ++ c = cmp(node, parent); + +-extern void rb_insert_color(struct rb_node *, struct rb_root *); +-extern void rb_erase(struct rb_node *, struct rb_root *); ++ if (c < 0) ++ link = &parent->rb_left; ++ else if (c > 0) ++ link = &parent->rb_right; ++ else ++ return parent; ++ } + +-/* Find logical next and previous nodes in a tree */ +-extern struct rb_node *rb_next(const struct rb_node *); +-extern struct rb_node *rb_prev(const struct rb_node *); +-extern struct rb_node *rb_first(const struct rb_root *); +-extern struct rb_node *rb_last(const struct rb_root *); ++ rb_link_node(node, parent, link); ++ rb_insert_color(node, tree); ++ return NULL; ++} + +-/* Fast replacement of a single node without remove/rebalance/add/rebalance */ +-extern void rb_replace_node(struct rb_node *victim, struct rb_node *new, +- struct rb_root *root); ++/** ++ * rb_find() - find @key in tree @tree ++ * @key: key to match ++ * @tree: tree to search ++ * @cmp: operator defining the node order ++ * ++ * Returns the rb_node matching @key or NULL. ++ */ ++static __always_inline struct rb_node * ++rb_find(const void *key, const struct rb_root *tree, ++ int (*cmp)(const void *key, const struct rb_node *)) ++{ ++ struct rb_node *node = tree->rb_node; ++ ++ while (node) { ++ int c = cmp(key, node); ++ ++ if (c < 0) ++ node = node->rb_left; ++ else if (c > 0) ++ node = node->rb_right; ++ else ++ return node; ++ } ++ ++ return NULL; ++} + +-static inline void rb_link_node(struct rb_node * node, struct rb_node * parent, +- struct rb_node ** rb_link) ++/** ++ * rb_find_first() - find the first @key in @tree ++ * @key: key to match ++ * @tree: tree to search ++ * @cmp: operator defining node order ++ * ++ * Returns the leftmost node matching @key, or NULL. ++ */ ++static __always_inline struct rb_node * ++rb_find_first(const void *key, const struct rb_root *tree, ++ int (*cmp)(const void *key, const struct rb_node *)) + { +- node->rb_parent_color = (unsigned long )parent; +- node->rb_left = node->rb_right = NULL; ++ struct rb_node *node = tree->rb_node; ++ struct rb_node *match = NULL; + +- *rb_link = node; ++ while (node) { ++ int c = cmp(key, node); ++ ++ if (c <= 0) { ++ if (!c) ++ match = node; ++ node = node->rb_left; ++ } else if (c > 0) { ++ node = node->rb_right; ++ } ++ } ++ ++ return match; + } + ++/** ++ * rb_next_match() - find the next @key in @tree ++ * @key: key to match ++ * @tree: tree to search ++ * @cmp: operator defining node order ++ * ++ * Returns the next node matching @key, or NULL. ++ */ ++static __always_inline struct rb_node * ++rb_next_match(const void *key, struct rb_node *node, ++ int (*cmp)(const void *key, const struct rb_node *)) ++{ ++ node = rb_next(node); ++ if (node && cmp(key, node)) ++ node = NULL; ++ return node; ++} ++ ++/** ++ * rb_for_each() - iterates a subtree matching @key ++ * @node: iterator ++ * @key: key to match ++ * @tree: tree to search ++ * @cmp: operator defining node order ++ */ ++#define rb_for_each(node, key, tree, cmp) \ ++ for ((node) = rb_find_first((key), (tree), (cmp)); \ ++ (node); (node) = rb_next_match((key), (node), (cmp))) ++ + #endif /* _LINUX_RBTREE_H */ +diff --git a/dlm_controld/rbtree_augmented.h b/dlm_controld/rbtree_augmented.h +new file mode 100644 +index 0000000..580a4c5 +--- /dev/null ++++ b/dlm_controld/rbtree_augmented.h +@@ -0,0 +1,303 @@ ++/* Copied from linux/include/linux/rbtree_augmented.h */ ++/* ++ Red Black Trees ++ (C) 1999 Andrea Arcangeli ++ (C) 2002 David Woodhouse ++ (C) 2012 Michel Lespinasse ++ ++ ++ linux/include/linux/rbtree_augmented.h ++*/ ++ ++#ifndef _LINUX_RBTREE_AUGMENTED_H ++#define _LINUX_RBTREE_AUGMENTED_H ++ ++#include "rbtree.h" ++#include "linux_helpers.h" ++ ++/* ++ * Please note - only struct rb_augment_callbacks and the prototypes for ++ * rb_insert_augmented() and rb_erase_augmented() are intended to be public. ++ * The rest are implementation details you are not expected to depend on. ++ * ++ * See Documentation/core-api/rbtree.rst for documentation and samples. ++ */ ++ ++struct rb_augment_callbacks { ++ void (*propagate)(struct rb_node *node, struct rb_node *stop); ++ void (*copy)(struct rb_node *old, struct rb_node *new); ++ void (*rotate)(struct rb_node *old, struct rb_node *new); ++}; ++ ++extern void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, ++ void (*augment_rotate)(struct rb_node *old, struct rb_node *new)); ++ ++/* ++ * Fixup the rbtree and update the augmented information when rebalancing. ++ * ++ * On insertion, the user must update the augmented information on the path ++ * leading to the inserted node, then call rb_link_node() as usual and ++ * rb_insert_augmented() instead of the usual rb_insert_color() call. ++ * If rb_insert_augmented() rebalances the rbtree, it will callback into ++ * a user provided function to update the augmented information on the ++ * affected subtrees. ++ */ ++static inline void ++rb_insert_augmented(struct rb_node *node, struct rb_root *root, ++ const struct rb_augment_callbacks *augment) ++{ ++ __rb_insert_augmented(node, root, augment->rotate); ++} ++ ++static inline void ++rb_insert_augmented_cached(struct rb_node *node, ++ struct rb_root_cached *root, bool newleft, ++ const struct rb_augment_callbacks *augment) ++{ ++ if (newleft) ++ root->rb_leftmost = node; ++ rb_insert_augmented(node, &root->rb_root, augment); ++} ++ ++/* ++ * Template for declaring augmented rbtree callbacks (generic case) ++ * ++ * RBSTATIC: 'static' or empty ++ * RBNAME: name of the rb_augment_callbacks structure ++ * RBSTRUCT: struct type of the tree nodes ++ * RBFIELD: name of struct rb_node field within RBSTRUCT ++ * RBAUGMENTED: name of field within RBSTRUCT holding data for subtree ++ * RBCOMPUTE: name of function that recomputes the RBAUGMENTED data ++ */ ++ ++#define RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ ++ RBSTRUCT, RBFIELD, RBAUGMENTED, RBCOMPUTE) \ ++static inline void \ ++RBNAME ## _propagate(struct rb_node *rb, struct rb_node *stop) \ ++{ \ ++ while (rb != stop) { \ ++ RBSTRUCT *node = rb_entry(rb, RBSTRUCT, RBFIELD); \ ++ if (RBCOMPUTE(node, true)) \ ++ break; \ ++ rb = rb_parent(&node->RBFIELD); \ ++ } \ ++} \ ++static inline void \ ++RBNAME ## _copy(struct rb_node *rb_old, struct rb_node *rb_new) \ ++{ \ ++ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ ++ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ ++ new->RBAUGMENTED = old->RBAUGMENTED; \ ++} \ ++static void \ ++RBNAME ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \ ++{ \ ++ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ ++ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ ++ new->RBAUGMENTED = old->RBAUGMENTED; \ ++ RBCOMPUTE(old, false); \ ++} \ ++RBSTATIC const struct rb_augment_callbacks RBNAME = { \ ++ .propagate = RBNAME ## _propagate, \ ++ .copy = RBNAME ## _copy, \ ++ .rotate = RBNAME ## _rotate \ ++}; ++ ++/* ++ * Template for declaring augmented rbtree callbacks, ++ * computing RBAUGMENTED scalar as max(RBCOMPUTE(node)) for all subtree nodes. ++ * ++ * RBSTATIC: 'static' or empty ++ * RBNAME: name of the rb_augment_callbacks structure ++ * RBSTRUCT: struct type of the tree nodes ++ * RBFIELD: name of struct rb_node field within RBSTRUCT ++ * RBTYPE: type of the RBAUGMENTED field ++ * RBAUGMENTED: name of RBTYPE field within RBSTRUCT holding data for subtree ++ * RBCOMPUTE: name of function that returns the per-node RBTYPE scalar ++ */ ++ ++#define RB_DECLARE_CALLBACKS_MAX(RBSTATIC, RBNAME, RBSTRUCT, RBFIELD, \ ++ RBTYPE, RBAUGMENTED, RBCOMPUTE) \ ++static inline bool RBNAME ## _compute_max(RBSTRUCT *node, bool exit) \ ++{ \ ++ RBSTRUCT *child; \ ++ RBTYPE max = RBCOMPUTE(node); \ ++ if (node->RBFIELD.rb_left) { \ ++ child = rb_entry(node->RBFIELD.rb_left, RBSTRUCT, RBFIELD); \ ++ if (child->RBAUGMENTED > max) \ ++ max = child->RBAUGMENTED; \ ++ } \ ++ if (node->RBFIELD.rb_right) { \ ++ child = rb_entry(node->RBFIELD.rb_right, RBSTRUCT, RBFIELD); \ ++ if (child->RBAUGMENTED > max) \ ++ max = child->RBAUGMENTED; \ ++ } \ ++ if (exit && node->RBAUGMENTED == max) \ ++ return true; \ ++ node->RBAUGMENTED = max; \ ++ return false; \ ++} \ ++RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ ++ RBSTRUCT, RBFIELD, RBAUGMENTED, RBNAME ## _compute_max) ++ ++ ++#define RB_RED 0 ++#define RB_BLACK 1 ++ ++#define __rb_parent(pc) ((struct rb_node *)(pc & ~3)) ++ ++#define __rb_color(pc) ((pc) & 1) ++#define __rb_is_black(pc) __rb_color(pc) ++#define __rb_is_red(pc) (!__rb_color(pc)) ++#define rb_color(rb) __rb_color((rb)->__rb_parent_color) ++#define rb_is_red(rb) __rb_is_red((rb)->__rb_parent_color) ++#define rb_is_black(rb) __rb_is_black((rb)->__rb_parent_color) ++ ++static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p) ++{ ++ rb->__rb_parent_color = rb_color(rb) + (unsigned long)p; ++} ++ ++static inline void rb_set_parent_color(struct rb_node *rb, ++ struct rb_node *p, int color) ++{ ++ rb->__rb_parent_color = (unsigned long)p + color; ++} ++ ++static inline void ++__rb_change_child(struct rb_node *old, struct rb_node *new, ++ struct rb_node *parent, struct rb_root *root) ++{ ++ if (parent) { ++ if (parent->rb_left == old) ++ WRITE_ONCE(parent->rb_left, new); ++ else ++ WRITE_ONCE(parent->rb_right, new); ++ } else ++ WRITE_ONCE(root->rb_node, new); ++} ++ ++extern void __rb_erase_color(struct rb_node *parent, struct rb_root *root, ++ void (*augment_rotate)(struct rb_node *old, struct rb_node *new)); ++ ++static __always_inline struct rb_node * ++__rb_erase_augmented(struct rb_node *node, struct rb_root *root, ++ const struct rb_augment_callbacks *augment) ++{ ++ struct rb_node *child = node->rb_right; ++ struct rb_node *tmp = node->rb_left; ++ struct rb_node *parent, *rebalance; ++ unsigned long pc; ++ ++ if (!tmp) { ++ /* ++ * Case 1: node to erase has no more than 1 child (easy!) ++ * ++ * Note that if there is one child it must be red due to 5) ++ * and node must be black due to 4). We adjust colors locally ++ * so as to bypass __rb_erase_color() later on. ++ */ ++ pc = node->__rb_parent_color; ++ parent = __rb_parent(pc); ++ __rb_change_child(node, child, parent, root); ++ if (child) { ++ child->__rb_parent_color = pc; ++ rebalance = NULL; ++ } else ++ rebalance = __rb_is_black(pc) ? parent : NULL; ++ tmp = parent; ++ } else if (!child) { ++ /* Still case 1, but this time the child is node->rb_left */ ++ tmp->__rb_parent_color = pc = node->__rb_parent_color; ++ parent = __rb_parent(pc); ++ __rb_change_child(node, tmp, parent, root); ++ rebalance = NULL; ++ tmp = parent; ++ } else { ++ struct rb_node *successor = child, *child2; ++ ++ tmp = child->rb_left; ++ if (!tmp) { ++ /* ++ * Case 2: node's successor is its right child ++ * ++ * (n) (s) ++ * / \ / \ ++ * (x) (s) -> (x) (c) ++ * \ ++ * (c) ++ */ ++ parent = successor; ++ child2 = successor->rb_right; ++ ++ augment->copy(node, successor); ++ } else { ++ /* ++ * Case 3: node's successor is leftmost under ++ * node's right child subtree ++ * ++ * (n) (s) ++ * / \ / \ ++ * (x) (y) -> (x) (y) ++ * / / ++ * (p) (p) ++ * / / ++ * (s) (c) ++ * \ ++ * (c) ++ */ ++ do { ++ parent = successor; ++ successor = tmp; ++ tmp = tmp->rb_left; ++ } while (tmp); ++ child2 = successor->rb_right; ++ WRITE_ONCE(parent->rb_left, child2); ++ WRITE_ONCE(successor->rb_right, child); ++ rb_set_parent(child, successor); ++ ++ augment->copy(node, successor); ++ augment->propagate(parent, successor); ++ } ++ ++ tmp = node->rb_left; ++ WRITE_ONCE(successor->rb_left, tmp); ++ rb_set_parent(tmp, successor); ++ ++ pc = node->__rb_parent_color; ++ tmp = __rb_parent(pc); ++ __rb_change_child(node, successor, tmp, root); ++ ++ if (child2) { ++ rb_set_parent_color(child2, parent, RB_BLACK); ++ rebalance = NULL; ++ } else { ++ rebalance = rb_is_black(successor) ? parent : NULL; ++ } ++ successor->__rb_parent_color = pc; ++ tmp = successor; ++ } ++ ++ augment->propagate(tmp, NULL); ++ return rebalance; ++} ++ ++static __always_inline void ++rb_erase_augmented(struct rb_node *node, struct rb_root *root, ++ const struct rb_augment_callbacks *augment) ++{ ++ struct rb_node *rebalance = __rb_erase_augmented(node, root, augment); ++ if (rebalance) ++ __rb_erase_color(rebalance, root, augment->rotate); ++} ++ ++static __always_inline void ++rb_erase_augmented_cached(struct rb_node *node, struct rb_root_cached *root, ++ const struct rb_augment_callbacks *augment) ++{ ++ if (root->rb_leftmost == node) ++ root->rb_leftmost = rb_next(node); ++ rb_erase_augmented(node, &root->rb_root, augment); ++} ++ ++#endif /* _LINUX_RBTREE_AUGMENTED_H */ +diff --git a/dlm_controld/rbtree_types.h b/dlm_controld/rbtree_types.h +new file mode 100644 +index 0000000..d6ed904 +--- /dev/null ++++ b/dlm_controld/rbtree_types.h +@@ -0,0 +1,34 @@ ++/* Copied from linux/include/linux/rbtree_types.h */ ++#ifndef _LINUX_RBTREE_TYPES_H ++#define _LINUX_RBTREE_TYPES_H ++ ++struct rb_node { ++ unsigned long __rb_parent_color; ++ struct rb_node *rb_right; ++ struct rb_node *rb_left; ++} __attribute__((aligned(sizeof(long)))); ++/* The alignment might seem pointless, but allegedly CRIS needs it */ ++ ++struct rb_root { ++ struct rb_node *rb_node; ++}; ++ ++/* ++ * Leftmost-cached rbtrees. ++ * ++ * We do not cache the rightmost node based on footprint ++ * size vs number of potential users that could benefit ++ * from O(1) rb_last(). Just not worth it, users that want ++ * this feature can always implement the logic explicitly. ++ * Furthermore, users that want to cache both pointers may ++ * find it a bit asymmetric, but that's ok. ++ */ ++struct rb_root_cached { ++ struct rb_root rb_root; ++ struct rb_node *rb_leftmost; ++}; ++ ++#define RB_ROOT (struct rb_root) { NULL, } ++#define RB_ROOT_CACHED (struct rb_root_cached) { {NULL, }, NULL } ++ ++#endif + +From 0ed5e3cbd812fa1e543f13076520a048adeda5f6 Mon Sep 17 00:00:00 2001 +From: Alexander Aring +Date: Aug 21 2023 16:19:52 +0000 +Subject: [PATCH 2/3] dlm_controld: update container_of() implementation + + +This patch updates the container_of() implementation taken from the +Linux kernel. + +We need to turn off -Wpointer-arith as the new container_of() +implementation does void pointer artihmetic as shown by this example +warning: + +linux_helpers.h:43:26: warning: pointer of type ‘void *’ used in +arithmetic [-Wpointer-arith] + +--- + +diff --git a/dlm_controld/Makefile b/dlm_controld/Makefile +index 28b6bb0..328a64b 100644 +--- a/dlm_controld/Makefile ++++ b/dlm_controld/Makefile +@@ -38,7 +38,7 @@ LIB_SOURCE = lib.c + + CFLAGS += -D_GNU_SOURCE -O2 -ggdb \ + -Wall -Wformat -Wformat-security -Wmissing-prototypes -Wnested-externs \ +- -Wpointer-arith -Wextra -Wshadow -Wcast-align -Wwrite-strings \ ++ -Wextra -Wshadow -Wcast-align -Wwrite-strings \ + -Waggregate-return -Wstrict-prototypes -Winline -Wredundant-decls \ + -Wno-sign-compare -Wno-unused-parameter -Wp,-D_FORTIFY_SOURCE=2 \ + -fexceptions -fasynchronous-unwind-tables -fdiagnostics-show-option \ +diff --git a/dlm_controld/linux_helpers.h b/dlm_controld/linux_helpers.h +index 5ef1346..09705cf 100644 +--- a/dlm_controld/linux_helpers.h ++++ b/dlm_controld/linux_helpers.h +@@ -3,6 +3,42 @@ + #ifndef __DLM_LINUX_HELPERS__ + #define __DLM_LINUX_HELPERS__ + ++/* ++ * static_assert - check integer constant expression at build time ++ * ++ * static_assert() is a wrapper for the C11 _Static_assert, with a ++ * little macro magic to make the message optional (defaulting to the ++ * stringification of the tested expression). ++ * ++ * Contrary to BUILD_BUG_ON(), static_assert() can be used at global ++ * scope, but requires the expression to be an integer constant ++ * expression (i.e., it is not enough that __builtin_constant_p() is ++ * true for expr). ++ * ++ * Also note that BUILD_BUG_ON() fails the build if the condition is ++ * true, while static_assert() fails the build if the expression is ++ * false. ++ */ ++#define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr) ++#define __static_assert(expr, msg, ...) _Static_assert(expr, msg) ++ ++#define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b)) ++ ++/** ++ * container_of - cast a member of a structure out to the containing structure ++ * @ptr: the pointer to the member. ++ * @type: the type of the container struct this is embedded in. ++ * @member: the name of the member within the struct. ++ * ++ * WARNING: any const qualifier of @ptr is lost. ++ */ ++#define container_of(ptr, type, member) ({ \ ++ void *__mptr = (void *)(ptr); \ ++ static_assert(__same_type(*(ptr), ((type *)0)->member) || \ ++ __same_type(*(ptr), void), \ ++ "pointer type mismatch in container_of()"); \ ++ ((type *)(__mptr - offsetof(type, member))); }) ++ + #define WRITE_ONCE(x, val) \ + do { \ + *(volatile typeof(x) *)&(x) = (val); \ +diff --git a/dlm_controld/list.h b/dlm_controld/list.h +index a2a5e5f..e9df2ef 100644 +--- a/dlm_controld/list.h ++++ b/dlm_controld/list.h +@@ -3,17 +3,7 @@ + #ifndef _LINUX_LIST_H + #define _LINUX_LIST_H + +-/** +- * container_of - cast a member of a structure out to the containing structure +- * +- * @ptr: the pointer to the member. +- * @type: the type of the container struct this is embedded in. +- * @member: the name of the member within the struct. +- * +- */ +-#define container_of(ptr, type, member) ({ \ +- const typeof( ((type *)0)->member ) *__mptr = (ptr); \ +- (type *)( (char *)__mptr - offsetof(type,member) );}) ++#include "linux_helpers.h" + + /* + * These are non-NULL pointers that will result in page faults +diff --git a/dlm_controld/rbtree.h b/dlm_controld/rbtree.h +index ddb86ff..48b6e32 100644 +--- a/dlm_controld/rbtree.h ++++ b/dlm_controld/rbtree.h +@@ -20,7 +20,7 @@ + #include + #include + +-#include "list.h" ++#include "linux_helpers.h" + #include "rbtree_types.h" + + #define rb_parent(r) ((struct rb_node *)((r)->__rb_parent_color & ~3)) + +From 8bbb1b0087d46e6d84eb80fb858e0dbe153dfa68 Mon Sep 17 00:00:00 2001 +From: Alexander Aring +Date: Aug 21 2023 16:20:08 +0000 +Subject: [PATCH 3/3] dlm_controld: update list implementation + + +This patch updates the list implementation taken from the Linux kernel. +There are new list manipulation functions introduced that could be +became useful later. + +--- + +diff --git a/dlm_controld/linux_helpers.h b/dlm_controld/linux_helpers.h +index 09705cf..f959cf5 100644 +--- a/dlm_controld/linux_helpers.h ++++ b/dlm_controld/linux_helpers.h +@@ -24,6 +24,11 @@ + + #define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b)) + ++#define POISON_POINTER_DELTA 0xdeadbeef ++ ++#define LIST_POISON1 ((void *) 0x100 + POISON_POINTER_DELTA) ++#define LIST_POISON2 ((void *) 0x122 + POISON_POINTER_DELTA) ++ + /** + * container_of - cast a member of a structure out to the containing structure + * @ptr: the pointer to the member. +@@ -39,6 +44,8 @@ + "pointer type mismatch in container_of()"); \ + ((type *)(__mptr - offsetof(type, member))); }) + ++#define READ_ONCE(x) (*(const volatile typeof(x) *)&(x)) ++ + #define WRITE_ONCE(x, val) \ + do { \ + *(volatile typeof(x) *)&(x) = (val); \ +diff --git a/dlm_controld/list.h b/dlm_controld/list.h +index e9df2ef..aab3b2b 100644 +--- a/dlm_controld/list.h ++++ b/dlm_controld/list.h +@@ -1,20 +1,17 @@ +-/* Copied from include/linux/list.h */ +- ++/* Copied from include/linux/list.h */ + #ifndef _LINUX_LIST_H + #define _LINUX_LIST_H + ++#include ++ + #include "linux_helpers.h" + +-/* +- * These are non-NULL pointers that will result in page faults +- * under normal circumstances, used to verify that nobody uses +- * non-initialized list entries. +- */ +-#define LIST_POISON1 ((void *) 0x00100100) +-#define LIST_POISON2 ((void *) 0x00200200) ++struct list_head { ++ struct list_head *next, *prev; ++}; + + /* +- * Simple doubly linked list implementation. ++ * Circular doubly linked list implementation. + * + * Some of the internal functions ("__xxx") are useful when + * manipulating whole lists rather than single entries, as +@@ -23,21 +20,44 @@ + * using the generic single-entry routines. + */ + +-struct list_head { +- struct list_head *next, *prev; +-}; +- + #define LIST_HEAD_INIT(name) { &(name), &(name) } + + #define LIST_HEAD(name) \ + struct list_head name = LIST_HEAD_INIT(name) + +-#define INIT_LIST_HEAD(ptr) do { \ +- (ptr)->next = (ptr); (ptr)->prev = (ptr); \ +-} while (0) ++/** ++ * INIT_LIST_HEAD - Initialize a list_head structure ++ * @list: list_head structure to be initialized. ++ * ++ * Initializes the list_head to point to itself. If it is a list header, ++ * the result is an empty list. ++ */ ++static inline void INIT_LIST_HEAD(struct list_head *list) ++{ ++ WRITE_ONCE(list->next, list); ++ WRITE_ONCE(list->prev, list); ++} ++ ++#ifdef CONFIG_DEBUG_LIST ++extern bool __list_add_valid(struct list_head *new, ++ struct list_head *prev, ++ struct list_head *next); ++extern bool __list_del_entry_valid(struct list_head *entry); ++#else ++static inline bool __list_add_valid(struct list_head *new, ++ struct list_head *prev, ++ struct list_head *next) ++{ ++ return true; ++} ++static inline bool __list_del_entry_valid(struct list_head *entry) ++{ ++ return true; ++} ++#endif + + /* +- * Insert a new entry between two known consecutive entries. ++ * Insert a new entry between two known consecutive entries. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! +@@ -46,10 +66,13 @@ static inline void __list_add(struct list_head *new, + struct list_head *prev, + struct list_head *next) + { ++ if (!__list_add_valid(new, prev, next)) ++ return; ++ + next->prev = new; + new->next = next; + new->prev = prev; +- prev->next = new; ++ WRITE_ONCE(prev->next, new); + } + + /** +@@ -65,6 +88,7 @@ static inline void list_add(struct list_head *new, struct list_head *head) + __list_add(new, head, head->next); + } + ++ + /** + * list_add_tail - add a new entry + * @new: new entry to be added +@@ -88,30 +112,99 @@ static inline void list_add_tail(struct list_head *new, struct list_head *head) + static inline void __list_del(struct list_head * prev, struct list_head * next) + { + next->prev = prev; +- prev->next = next; ++ WRITE_ONCE(prev->next, next); ++} ++ ++/* ++ * Delete a list entry and clear the 'prev' pointer. ++ * ++ * This is a special-purpose list clearing method used in the networking code ++ * for lists allocated as per-cpu, where we don't want to incur the extra ++ * WRITE_ONCE() overhead of a regular list_del_init(). The code that uses this ++ * needs to check the node 'prev' pointer instead of calling list_empty(). ++ */ ++static inline void __list_del_clearprev(struct list_head *entry) ++{ ++ __list_del(entry->prev, entry->next); ++ entry->prev = NULL; ++} ++ ++static inline void __list_del_entry(struct list_head *entry) ++{ ++ if (!__list_del_entry_valid(entry)) ++ return; ++ ++ __list_del(entry->prev, entry->next); + } + + /** + * list_del - deletes entry from list. + * @entry: the element to delete from the list. +- * Note: list_empty on entry does not return true after this, the entry is ++ * Note: list_empty() on entry does not return true after this, the entry is + * in an undefined state. + */ + static inline void list_del(struct list_head *entry) + { +- __list_del(entry->prev, entry->next); ++ __list_del_entry(entry); + entry->next = LIST_POISON1; + entry->prev = LIST_POISON2; + } + + /** ++ * list_replace - replace old entry by new one ++ * @old : the element to be replaced ++ * @new : the new element to insert ++ * ++ * If @old was empty, it will be overwritten. ++ */ ++static inline void list_replace(struct list_head *old, ++ struct list_head *new) ++{ ++ new->next = old->next; ++ new->next->prev = new; ++ new->prev = old->prev; ++ new->prev->next = new; ++} ++ ++/** ++ * list_replace_init - replace old entry by new one and initialize the old one ++ * @old : the element to be replaced ++ * @new : the new element to insert ++ * ++ * If @old was empty, it will be overwritten. ++ */ ++static inline void list_replace_init(struct list_head *old, ++ struct list_head *new) ++{ ++ list_replace(old, new); ++ INIT_LIST_HEAD(old); ++} ++ ++/** ++ * list_swap - replace entry1 with entry2 and re-add entry1 at entry2's position ++ * @entry1: the location to place entry2 ++ * @entry2: the location to place entry1 ++ */ ++static inline void list_swap(struct list_head *entry1, ++ struct list_head *entry2) ++{ ++ struct list_head *pos = entry2->prev; ++ ++ list_del(entry2); ++ list_replace(entry1, entry2); ++ if (pos == entry1) ++ pos = entry2; ++ list_add(entry1, pos); ++} ++ ++/** + * list_del_init - deletes entry from list and reinitialize it. + * @entry: the element to delete from the list. + */ + static inline void list_del_init(struct list_head *entry) + { +- __list_del(entry->prev, entry->next); +- INIT_LIST_HEAD(entry); ++ __list_del_entry(entry); ++ INIT_LIST_HEAD(entry); + } + + /** +@@ -121,8 +214,8 @@ static inline void list_del_init(struct list_head *entry) + */ + static inline void list_move(struct list_head *list, struct list_head *head) + { +- __list_del(list->prev, list->next); +- list_add(list, head); ++ __list_del_entry(list); ++ list_add(list, head); + } + + /** +@@ -133,8 +226,61 @@ static inline void list_move(struct list_head *list, struct list_head *head) + static inline void list_move_tail(struct list_head *list, + struct list_head *head) + { +- __list_del(list->prev, list->next); +- list_add_tail(list, head); ++ __list_del_entry(list); ++ list_add_tail(list, head); ++} ++ ++/** ++ * list_bulk_move_tail - move a subsection of a list to its tail ++ * @head: the head that will follow our entry ++ * @first: first entry to move ++ * @last: last entry to move, can be the same as first ++ * ++ * Move all entries between @first and including @last before @head. ++ * All three entries must belong to the same linked list. ++ */ ++static inline void list_bulk_move_tail(struct list_head *head, ++ struct list_head *first, ++ struct list_head *last) ++{ ++ first->prev->next = last->next; ++ last->next->prev = first->prev; ++ ++ head->prev->next = first; ++ first->prev = head->prev; ++ ++ last->next = head; ++ head->prev = last; ++} ++ ++/** ++ * list_is_first -- tests whether @list is the first entry in list @head ++ * @list: the entry to test ++ * @head: the head of the list ++ */ ++static inline int list_is_first(const struct list_head *list, const struct list_head *head) ++{ ++ return list->prev == head; ++} ++ ++/** ++ * list_is_last - tests whether @list is the last entry in list @head ++ * @list: the entry to test ++ * @head: the head of the list ++ */ ++static inline int list_is_last(const struct list_head *list, const struct list_head *head) ++{ ++ return list->next == head; ++} ++ ++/** ++ * list_is_head - tests whether @list is the list @head ++ * @list: the entry to test ++ * @head: the head of the list ++ */ ++static inline int list_is_head(const struct list_head *list, const struct list_head *head) ++{ ++ return list == head; + } + + /** +@@ -143,50 +289,155 @@ static inline void list_move_tail(struct list_head *list, + */ + static inline int list_empty(const struct list_head *head) + { +- return head->next == head; ++ return READ_ONCE(head->next) == head; + } + + /** +- * list_empty_careful - tests whether a list is +- * empty _and_ checks that no other CPU might be +- * in the process of still modifying either member +- * +- * NOTE: using list_empty_careful() without synchronization +- * can only be safe if the only activity that can happen +- * to the list entry is list_del_init(). Eg. it cannot be used +- * if another CPU could re-list_add() it. ++ * list_rotate_left - rotate the list to the left ++ * @head: the head of the list ++ */ ++static inline void list_rotate_left(struct list_head *head) ++{ ++ struct list_head *first; ++ ++ if (!list_empty(head)) { ++ first = head->next; ++ list_move_tail(first, head); ++ } ++} ++ ++/** ++ * list_rotate_to_front() - Rotate list to specific item. ++ * @list: The desired new front of the list. ++ * @head: The head of the list. + * ++ * Rotates list so that @list becomes the new front of the list. ++ */ ++static inline void list_rotate_to_front(struct list_head *list, ++ struct list_head *head) ++{ ++ /* ++ * Deletes the list head from the list denoted by @head and ++ * places it as the tail of @list, this effectively rotates the ++ * list so that @list is at the front. ++ */ ++ list_move_tail(head, list); ++} ++ ++/** ++ * list_is_singular - tests whether a list has just one entry. + * @head: the list to test. + */ +-static inline int list_empty_careful(const struct list_head *head) ++static inline int list_is_singular(const struct list_head *head) + { +- struct list_head *next = head->next; +- return (next == head) && (next == head->prev); ++ return !list_empty(head) && (head->next == head->prev); + } + +-static inline void __list_splice(struct list_head *list, +- struct list_head *head) ++static inline void __list_cut_position(struct list_head *list, ++ struct list_head *head, struct list_head *entry) ++{ ++ struct list_head *new_first = entry->next; ++ list->next = head->next; ++ list->next->prev = list; ++ list->prev = entry; ++ entry->next = list; ++ head->next = new_first; ++ new_first->prev = head; ++} ++ ++/** ++ * list_cut_position - cut a list into two ++ * @list: a new list to add all removed entries ++ * @head: a list with entries ++ * @entry: an entry within head, could be the head itself ++ * and if so we won't cut the list ++ * ++ * This helper moves the initial part of @head, up to and ++ * including @entry, from @head to @list. You should ++ * pass on @entry an element you know is on @head. @list ++ * should be an empty list or a list you do not care about ++ * losing its data. ++ * ++ */ ++static inline void list_cut_position(struct list_head *list, ++ struct list_head *head, struct list_head *entry) ++{ ++ if (list_empty(head)) ++ return; ++ if (list_is_singular(head) && !list_is_head(entry, head) && (entry != head->next)) ++ return; ++ if (list_is_head(entry, head)) ++ INIT_LIST_HEAD(list); ++ else ++ __list_cut_position(list, head, entry); ++} ++ ++/** ++ * list_cut_before - cut a list into two, before given entry ++ * @list: a new list to add all removed entries ++ * @head: a list with entries ++ * @entry: an entry within head, could be the head itself ++ * ++ * This helper moves the initial part of @head, up to but ++ * excluding @entry, from @head to @list. You should pass ++ * in @entry an element you know is on @head. @list should ++ * be an empty list or a list you do not care about losing ++ * its data. ++ * If @entry == @head, all entries on @head are moved to ++ * @list. ++ */ ++static inline void list_cut_before(struct list_head *list, ++ struct list_head *head, ++ struct list_head *entry) ++{ ++ if (head->next == entry) { ++ INIT_LIST_HEAD(list); ++ return; ++ } ++ list->next = head->next; ++ list->next->prev = list; ++ list->prev = entry->prev; ++ list->prev->next = list; ++ head->next = entry; ++ entry->prev = head; ++} ++ ++static inline void __list_splice(const struct list_head *list, ++ struct list_head *prev, ++ struct list_head *next) + { + struct list_head *first = list->next; + struct list_head *last = list->prev; +- struct list_head *at = head->next; + +- first->prev = head; +- head->next = first; ++ first->prev = prev; ++ prev->next = first; + +- last->next = at; +- at->prev = last; ++ last->next = next; ++ next->prev = last; ++} ++ ++/** ++ * list_splice - join two lists, this is designed for stacks ++ * @list: the new list to add. ++ * @head: the place to add it in the first list. ++ */ ++static inline void list_splice(const struct list_head *list, ++ struct list_head *head) ++{ ++ if (!list_empty(list)) ++ __list_splice(list, head, head->next); + } + + /** +- * list_splice - join two lists ++ * list_splice_tail - join two lists, each list being a queue + * @list: the new list to add. + * @head: the place to add it in the first list. + */ +-static inline void list_splice(struct list_head *list, struct list_head *head) ++static inline void list_splice_tail(struct list_head *list, ++ struct list_head *head) + { + if (!list_empty(list)) +- __list_splice(list, head); ++ __list_splice(list, head->prev, head); + } + + /** +@@ -200,7 +451,24 @@ static inline void list_splice_init(struct list_head *list, + struct list_head *head) + { + if (!list_empty(list)) { +- __list_splice(list, head); ++ __list_splice(list, head, head->next); ++ INIT_LIST_HEAD(list); ++ } ++} ++ ++/** ++ * list_splice_tail_init - join two lists and reinitialise the emptied list ++ * @list: the new list to add. ++ * @head: the place to add it in the first list. ++ * ++ * Each of the lists is a queue. ++ * The list at @list is reinitialised ++ */ ++static inline void list_splice_tail_init(struct list_head *list, ++ struct list_head *head) ++{ ++ if (!list_empty(list)) { ++ __list_splice(list, head->prev, head); + INIT_LIST_HEAD(list); + } + } +@@ -209,16 +477,16 @@ static inline void list_splice_init(struct list_head *list, + * list_entry - get the struct for this entry + * @ptr: the &struct list_head pointer. + * @type: the type of the struct this is embedded in. +- * @member: the name of the list_struct within the struct. ++ * @member: the name of the list_head within the struct. + */ + #define list_entry(ptr, type, member) \ + container_of(ptr, type, member) + + /** + * list_first_entry - get the first element from a list +- * @ptr: the list head to take the element from. +- * @type: the type of the struct this is embedded in. +- * @member: the name of the list_struct within the struct. ++ * @ptr: the list head to take the element from. ++ * @type: the type of the struct this is embedded in. ++ * @member: the name of the list_head within the struct. + * + * Note, that list is expected to be not empty. + */ +@@ -226,100 +494,303 @@ static inline void list_splice_init(struct list_head *list, + list_entry((ptr)->next, type, member) + + /** ++ * list_last_entry - get the last element from a list ++ * @ptr: the list head to take the element from. ++ * @type: the type of the struct this is embedded in. ++ * @member: the name of the list_head within the struct. ++ * ++ * Note, that list is expected to be not empty. ++ */ ++#define list_last_entry(ptr, type, member) \ ++ list_entry((ptr)->prev, type, member) ++ ++/** ++ * list_first_entry_or_null - get the first element from a list ++ * @ptr: the list head to take the element from. ++ * @type: the type of the struct this is embedded in. ++ * @member: the name of the list_head within the struct. ++ * ++ * Note that if the list is empty, it returns NULL. ++ */ ++#define list_first_entry_or_null(ptr, type, member) ({ \ ++ struct list_head *head__ = (ptr); \ ++ struct list_head *pos__ = READ_ONCE(head__->next); \ ++ pos__ != head__ ? list_entry(pos__, type, member) : NULL; \ ++}) ++ ++/** ++ * list_next_entry - get the next element in list ++ * @pos: the type * to cursor ++ * @member: the name of the list_head within the struct. ++ */ ++#define list_next_entry(pos, member) \ ++ list_entry((pos)->member.next, typeof(*(pos)), member) ++ ++/** ++ * list_next_entry_circular - get the next element in list ++ * @pos: the type * to cursor. ++ * @head: the list head to take the element from. ++ * @member: the name of the list_head within the struct. ++ * ++ * Wraparound if pos is the last element (return the first element). ++ * Note, that list is expected to be not empty. ++ */ ++#define list_next_entry_circular(pos, head, member) \ ++ (list_is_last(&(pos)->member, head) ? \ ++ list_first_entry(head, typeof(*(pos)), member) : list_next_entry(pos, member)) ++ ++/** ++ * list_prev_entry - get the prev element in list ++ * @pos: the type * to cursor ++ * @member: the name of the list_head within the struct. ++ */ ++#define list_prev_entry(pos, member) \ ++ list_entry((pos)->member.prev, typeof(*(pos)), member) ++ ++/** ++ * list_prev_entry_circular - get the prev element in list ++ * @pos: the type * to cursor. ++ * @head: the list head to take the element from. ++ * @member: the name of the list_head within the struct. ++ * ++ * Wraparound if pos is the first element (return the last element). ++ * Note, that list is expected to be not empty. ++ */ ++#define list_prev_entry_circular(pos, head, member) \ ++ (list_is_first(&(pos)->member, head) ? \ ++ list_last_entry(head, typeof(*(pos)), member) : list_prev_entry(pos, member)) ++ ++/** + * list_for_each - iterate over a list +- * @pos: the &struct list_head to use as a loop counter. ++ * @pos: the &struct list_head to use as a loop cursor. + * @head: the head for your list. + */ + #define list_for_each(pos, head) \ +- for (pos = (head)->next; pos != (head); pos = pos->next) ++ for (pos = (head)->next; !list_is_head(pos, (head)); pos = pos->next) + + /** +- * __list_for_each - iterate over a list +- * @pos: the &struct list_head to use as a loop counter. ++ * list_for_each_continue - continue iteration over a list ++ * @pos: the &struct list_head to use as a loop cursor. + * @head: the head for your list. + * +- * This variant differs from list_for_each() in that it's the +- * simplest possible list iteration code, no prefetching is done. +- * Use this for code that knows the list to be very short (empty +- * or 1 entry) most of the time. ++ * Continue to iterate over a list, continuing after the current position. + */ +-#define __list_for_each(pos, head) \ +- for (pos = (head)->next; pos != (head); pos = pos->next) ++#define list_for_each_continue(pos, head) \ ++ for (pos = pos->next; !list_is_head(pos, (head)); pos = pos->next) + + /** + * list_for_each_prev - iterate over a list backwards +- * @pos: the &struct list_head to use as a loop counter. ++ * @pos: the &struct list_head to use as a loop cursor. + * @head: the head for your list. + */ + #define list_for_each_prev(pos, head) \ +- for (pos = (head)->prev; pos != (head); pos = pos->prev) +- ++ for (pos = (head)->prev; !list_is_head(pos, (head)); pos = pos->prev) ++ + /** +- * list_for_each_safe - iterate over a list safe against removal of list entry +- * @pos: the &struct list_head to use as a loop counter. ++ * list_for_each_safe - iterate over a list safe against removal of list entry ++ * @pos: the &struct list_head to use as a loop cursor. + * @n: another &struct list_head to use as temporary storage + * @head: the head for your list. + */ + #define list_for_each_safe(pos, n, head) \ +- for (pos = (head)->next, n = pos->next; pos != (head); \ +- pos = n, n = pos->next) ++ for (pos = (head)->next, n = pos->next; \ ++ !list_is_head(pos, (head)); \ ++ pos = n, n = pos->next) ++ ++/** ++ * list_for_each_prev_safe - iterate over a list backwards safe against removal of list entry ++ * @pos: the &struct list_head to use as a loop cursor. ++ * @n: another &struct list_head to use as temporary storage ++ * @head: the head for your list. ++ */ ++#define list_for_each_prev_safe(pos, n, head) \ ++ for (pos = (head)->prev, n = pos->prev; \ ++ !list_is_head(pos, (head)); \ ++ pos = n, n = pos->prev) ++ ++/** ++ * list_count_nodes - count nodes in the list ++ * @head: the head for your list. ++ */ ++static inline size_t list_count_nodes(struct list_head *head) ++{ ++ struct list_head *pos; ++ size_t count = 0; ++ ++ list_for_each(pos, head) ++ count++; ++ ++ return count; ++} ++ ++/** ++ * list_entry_is_head - test if the entry points to the head of the list ++ * @pos: the type * to cursor ++ * @head: the head for your list. ++ * @member: the name of the list_head within the struct. ++ */ ++#define list_entry_is_head(pos, head, member) \ ++ (&pos->member == (head)) + + /** + * list_for_each_entry - iterate over list of given type +- * @pos: the type * to use as a loop counter. ++ * @pos: the type * to use as a loop cursor. + * @head: the head for your list. +- * @member: the name of the list_struct within the struct. ++ * @member: the name of the list_head within the struct. + */ + #define list_for_each_entry(pos, head, member) \ +- for (pos = list_entry((head)->next, typeof(*pos), member); \ +- &pos->member != (head); \ +- pos = list_entry(pos->member.next, typeof(*pos), member)) ++ for (pos = list_first_entry(head, typeof(*pos), member); \ ++ !list_entry_is_head(pos, head, member); \ ++ pos = list_next_entry(pos, member)) + + /** + * list_for_each_entry_reverse - iterate backwards over list of given type. +- * @pos: the type * to use as a loop counter. ++ * @pos: the type * to use as a loop cursor. + * @head: the head for your list. +- * @member: the name of the list_struct within the struct. ++ * @member: the name of the list_head within the struct. + */ + #define list_for_each_entry_reverse(pos, head, member) \ +- for (pos = list_entry((head)->prev, typeof(*pos), member); \ +- &pos->member != (head); \ +- pos = list_entry(pos->member.prev, typeof(*pos), member)) ++ for (pos = list_last_entry(head, typeof(*pos), member); \ ++ !list_entry_is_head(pos, head, member); \ ++ pos = list_prev_entry(pos, member)) + + /** +- * list_prepare_entry - prepare a pos entry for use as a start point in +- * list_for_each_entry_continue ++ * list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue() + * @pos: the type * to use as a start point + * @head: the head of the list +- * @member: the name of the list_struct within the struct. ++ * @member: the name of the list_head within the struct. ++ * ++ * Prepares a pos entry for use as a start point in list_for_each_entry_continue(). + */ + #define list_prepare_entry(pos, head, member) \ + ((pos) ? : list_entry(head, typeof(*pos), member)) + + /** +- * list_for_each_entry_continue - iterate over list of given type +- * continuing after existing point +- * @pos: the type * to use as a loop counter. ++ * list_for_each_entry_continue - continue iteration over list of given type ++ * @pos: the type * to use as a loop cursor. + * @head: the head for your list. +- * @member: the name of the list_struct within the struct. ++ * @member: the name of the list_head within the struct. ++ * ++ * Continue to iterate over list of given type, continuing after ++ * the current position. + */ + #define list_for_each_entry_continue(pos, head, member) \ +- for (pos = list_entry(pos->member.next, typeof(*pos), member); \ +- &pos->member != (head); \ +- pos = list_entry(pos->member.next, typeof(*pos), member)) ++ for (pos = list_next_entry(pos, member); \ ++ !list_entry_is_head(pos, head, member); \ ++ pos = list_next_entry(pos, member)) ++ ++/** ++ * list_for_each_entry_continue_reverse - iterate backwards from the given point ++ * @pos: the type * to use as a loop cursor. ++ * @head: the head for your list. ++ * @member: the name of the list_head within the struct. ++ * ++ * Start to iterate over list of given type backwards, continuing after ++ * the current position. ++ */ ++#define list_for_each_entry_continue_reverse(pos, head, member) \ ++ for (pos = list_prev_entry(pos, member); \ ++ !list_entry_is_head(pos, head, member); \ ++ pos = list_prev_entry(pos, member)) ++ ++/** ++ * list_for_each_entry_from - iterate over list of given type from the current point ++ * @pos: the type * to use as a loop cursor. ++ * @head: the head for your list. ++ * @member: the name of the list_head within the struct. ++ * ++ * Iterate over list of given type, continuing from current position. ++ */ ++#define list_for_each_entry_from(pos, head, member) \ ++ for (; !list_entry_is_head(pos, head, member); \ ++ pos = list_next_entry(pos, member)) ++ ++/** ++ * list_for_each_entry_from_reverse - iterate backwards over list of given type ++ * from the current point ++ * @pos: the type * to use as a loop cursor. ++ * @head: the head for your list. ++ * @member: the name of the list_head within the struct. ++ * ++ * Iterate backwards over list of given type, continuing from current position. ++ */ ++#define list_for_each_entry_from_reverse(pos, head, member) \ ++ for (; !list_entry_is_head(pos, head, member); \ ++ pos = list_prev_entry(pos, member)) + + /** + * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry +- * @pos: the type * to use as a loop counter. ++ * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @head: the head for your list. +- * @member: the name of the list_struct within the struct. ++ * @member: the name of the list_head within the struct. + */ + #define list_for_each_entry_safe(pos, n, head, member) \ +- for (pos = list_entry((head)->next, typeof(*pos), member), \ +- n = list_entry(pos->member.next, typeof(*pos), member); \ +- &pos->member != (head); \ +- pos = n, n = list_entry(n->member.next, typeof(*n), member)) ++ for (pos = list_first_entry(head, typeof(*pos), member), \ ++ n = list_next_entry(pos, member); \ ++ !list_entry_is_head(pos, head, member); \ ++ pos = n, n = list_next_entry(n, member)) ++ ++/** ++ * list_for_each_entry_safe_continue - continue list iteration safe against removal ++ * @pos: the type * to use as a loop cursor. ++ * @n: another type * to use as temporary storage ++ * @head: the head for your list. ++ * @member: the name of the list_head within the struct. ++ * ++ * Iterate over list of given type, continuing after current point, ++ * safe against removal of list entry. ++ */ ++#define list_for_each_entry_safe_continue(pos, n, head, member) \ ++ for (pos = list_next_entry(pos, member), \ ++ n = list_next_entry(pos, member); \ ++ !list_entry_is_head(pos, head, member); \ ++ pos = n, n = list_next_entry(n, member)) + ++/** ++ * list_for_each_entry_safe_from - iterate over list from current point safe against removal ++ * @pos: the type * to use as a loop cursor. ++ * @n: another type * to use as temporary storage ++ * @head: the head for your list. ++ * @member: the name of the list_head within the struct. ++ * ++ * Iterate over list of given type from current point, safe against ++ * removal of list entry. ++ */ ++#define list_for_each_entry_safe_from(pos, n, head, member) \ ++ for (n = list_next_entry(pos, member); \ ++ !list_entry_is_head(pos, head, member); \ ++ pos = n, n = list_next_entry(n, member)) ++ ++/** ++ * list_for_each_entry_safe_reverse - iterate backwards over list safe against removal ++ * @pos: the type * to use as a loop cursor. ++ * @n: another type * to use as temporary storage ++ * @head: the head for your list. ++ * @member: the name of the list_head within the struct. ++ * ++ * Iterate backwards over list of given type, safe against removal ++ * of list entry. ++ */ ++#define list_for_each_entry_safe_reverse(pos, n, head, member) \ ++ for (pos = list_last_entry(head, typeof(*pos), member), \ ++ n = list_prev_entry(pos, member); \ ++ !list_entry_is_head(pos, head, member); \ ++ pos = n, n = list_prev_entry(n, member)) ++ ++/** ++ * list_safe_reset_next - reset a stale list_for_each_entry_safe loop ++ * @pos: the loop cursor used in the list_for_each_entry_safe loop ++ * @n: temporary storage used in list_for_each_entry_safe ++ * @member: the name of the list_head within the struct. ++ * ++ * list_safe_reset_next is not safe to use in general if the list may be ++ * modified concurrently (eg. the lock is dropped in the loop body). An ++ * exception to this is if the cursor element (pos) is pinned in the list, ++ * and list_safe_reset_next is called after re-taking the lock and before ++ * completing the current iteration of the loop body. ++ */ ++#define list_safe_reset_next(pos, n, member) \ ++ n = list_next_entry(pos, member) + + #endif +