1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_LIST_BL_H 3 #define _LINUX_LIST_BL_H 4 5 #include <linux/list.h> 6 #include <linux/bit_spinlock.h> 7 8 /* 9 * Special version of lists, where head of the list has a lock in the lowest 10 * bit. This is useful for scalable hash tables without increasing memory 11 * footprint overhead. 12 * 13 * For modification operations, the 0 bit of hlist_bl_head->first 14 * pointer must be set. 15 * 16 * With some small modifications, this can easily be adapted to store several 17 * arbitrary bits (not just a single lock bit), if the need arises to store 18 * some fast and compact auxiliary data. 19 */ 20 21 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) 22 #define LIST_BL_LOCKMASK 1UL 23 #else 24 #define LIST_BL_LOCKMASK 0UL 25 #endif 26 27 #ifdef CONFIG_DEBUG_LIST 28 #define LIST_BL_BUG_ON(x) BUG_ON(x) 29 #else 30 #define LIST_BL_BUG_ON(x) 31 #endif 32 33 34 struct hlist_bl_head { 35 struct hlist_bl_node *first; 36 }; 37 38 struct hlist_bl_node { 39 struct hlist_bl_node *next, **pprev; 40 }; 41 #define INIT_HLIST_BL_HEAD(ptr) \ 42 ((ptr)->first = NULL) 43 INIT_HLIST_BL_NODE(struct hlist_bl_node * h)44 static inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h) 45 { 46 h->next = NULL; 47 h->pprev = NULL; 48 } 49 50 #define hlist_bl_entry(ptr, type, member) container_of(ptr,type,member) 51 hlist_bl_unhashed(const struct hlist_bl_node * h)52 static inline bool hlist_bl_unhashed(const struct hlist_bl_node *h) 53 { 54 return !h->pprev; 55 } 56 hlist_bl_first(struct hlist_bl_head * h)57 static inline struct hlist_bl_node *hlist_bl_first(struct hlist_bl_head *h) 58 { 59 return (struct hlist_bl_node *) 60 ((unsigned long)h->first & ~LIST_BL_LOCKMASK); 61 } 62 hlist_bl_set_first(struct hlist_bl_head * h,struct hlist_bl_node * n)63 static inline void hlist_bl_set_first(struct hlist_bl_head *h, 64 struct hlist_bl_node *n) 65 { 66 LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK); 67 LIST_BL_BUG_ON(((unsigned long)h->first & LIST_BL_LOCKMASK) != 68 LIST_BL_LOCKMASK); 69 h->first = (struct hlist_bl_node *)((unsigned long)n | LIST_BL_LOCKMASK); 70 } 71 hlist_bl_empty(const struct hlist_bl_head * h)72 static inline bool hlist_bl_empty(const struct hlist_bl_head *h) 73 { 74 return !((unsigned long)READ_ONCE(h->first) & ~LIST_BL_LOCKMASK); 75 } 76 hlist_bl_add_head(struct hlist_bl_node * n,struct hlist_bl_head * h)77 static inline void hlist_bl_add_head(struct hlist_bl_node *n, 78 struct hlist_bl_head *h) 79 { 80 struct hlist_bl_node *first = hlist_bl_first(h); 81 82 n->next = first; 83 if (first) 84 first->pprev = &n->next; 85 n->pprev = &h->first; 86 hlist_bl_set_first(h, n); 87 } 88 hlist_bl_add_before(struct hlist_bl_node * n,struct hlist_bl_node * next)89 static inline void hlist_bl_add_before(struct hlist_bl_node *n, 90 struct hlist_bl_node *next) 91 { 92 struct hlist_bl_node **pprev = next->pprev; 93 94 n->pprev = pprev; 95 n->next = next; 96 next->pprev = &n->next; 97 98 /* pprev may be `first`, so be careful not to lose the lock bit */ 99 WRITE_ONCE(*pprev, 100 (struct hlist_bl_node *) 101 ((uintptr_t)n | ((uintptr_t)*pprev & LIST_BL_LOCKMASK))); 102 } 103 hlist_bl_add_behind(struct hlist_bl_node * n,struct hlist_bl_node * prev)104 static inline void hlist_bl_add_behind(struct hlist_bl_node *n, 105 struct hlist_bl_node *prev) 106 { 107 n->next = prev->next; 108 n->pprev = &prev->next; 109 prev->next = n; 110 111 if (n->next) 112 n->next->pprev = &n->next; 113 } 114 __hlist_bl_del(struct hlist_bl_node * n)115 static inline void __hlist_bl_del(struct hlist_bl_node *n) 116 { 117 struct hlist_bl_node *next = n->next; 118 struct hlist_bl_node **pprev = n->pprev; 119 120 LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK); 121 122 /* pprev may be `first`, so be careful not to lose the lock bit */ 123 WRITE_ONCE(*pprev, 124 (struct hlist_bl_node *) 125 ((unsigned long)next | 126 ((unsigned long)*pprev & LIST_BL_LOCKMASK))); 127 if (next) 128 next->pprev = pprev; 129 } 130 hlist_bl_del(struct hlist_bl_node * n)131 static inline void hlist_bl_del(struct hlist_bl_node *n) 132 { 133 __hlist_bl_del(n); 134 n->next = LIST_POISON1; 135 n->pprev = LIST_POISON2; 136 } 137 hlist_bl_del_init(struct hlist_bl_node * n)138 static inline void hlist_bl_del_init(struct hlist_bl_node *n) 139 { 140 if (!hlist_bl_unhashed(n)) { 141 __hlist_bl_del(n); 142 INIT_HLIST_BL_NODE(n); 143 } 144 } 145 hlist_bl_lock(struct hlist_bl_head * b)146 static inline void hlist_bl_lock(struct hlist_bl_head *b) 147 { 148 bit_spin_lock(0, (unsigned long *)b); 149 } 150 hlist_bl_unlock(struct hlist_bl_head * b)151 static inline void hlist_bl_unlock(struct hlist_bl_head *b) 152 { 153 __bit_spin_unlock(0, (unsigned long *)b); 154 } 155 hlist_bl_is_locked(struct hlist_bl_head * b)156 static inline bool hlist_bl_is_locked(struct hlist_bl_head *b) 157 { 158 return bit_spin_is_locked(0, (unsigned long *)b); 159 } 160 161 /** 162 * hlist_bl_for_each_entry - iterate over list of given type 163 * @tpos: the type * to use as a loop cursor. 164 * @pos: the &struct hlist_node to use as a loop cursor. 165 * @head: the head for your list. 166 * @member: the name of the hlist_node within the struct. 167 * 168 */ 169 #define hlist_bl_for_each_entry(tpos, pos, head, member) \ 170 for (pos = hlist_bl_first(head); \ 171 pos && \ 172 ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \ 173 pos = pos->next) 174 175 /** 176 * hlist_bl_for_each_entry_safe - iterate over list of given type safe against removal of list entry 177 * @tpos: the type * to use as a loop cursor. 178 * @pos: the &struct hlist_node to use as a loop cursor. 179 * @n: another &struct hlist_node to use as temporary storage 180 * @head: the head for your list. 181 * @member: the name of the hlist_node within the struct. 182 */ 183 #define hlist_bl_for_each_entry_safe(tpos, pos, n, head, member) \ 184 for (pos = hlist_bl_first(head); \ 185 pos && ({ n = pos->next; 1; }) && \ 186 ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \ 187 pos = n) 188 189 #endif 190