24 static void poison_error(mempool_t *pool, void *element, size_t size,  in poison_error()  argument
27 	const int nr = pool->curr_nr;  in poison_error()
33 	pr_err("Mempool %p size %zu\n", pool, size);  in poison_error()
41 static void __check_element(mempool_t *pool, void *element, size_t size)  in __check_element()  argument
50 			poison_error(pool, element, size, i);  in __check_element()
57 static void check_element(mempool_t *pool, void *element)  in check_element()  argument
64 	if (pool->free == mempool_kfree) {  in check_element()
65 		__check_element(pool, element, (size_t)pool->pool_data);  in check_element()
66 	} else if (pool->free == mempool_free_slab) {  in check_element()
67 		__check_element(pool, element, kmem_cache_size(pool->pool_data));  in check_element()
68 	} else if (pool->free == mempool_free_pages) {  in check_element()
70 		int order = (int)(long)pool->pool_data;  in check_element()
73 		__check_element(pool, addr, 1UL << (PAGE_SHIFT + order));  in check_element()
86 static void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
93 	if (pool->alloc == mempool_kmalloc) {  in poison_element()
94 		__poison_element(element, (size_t)pool->pool_data);  in poison_element()
95 	} else if (pool->alloc == mempool_alloc_slab) {  in poison_element()
96 		__poison_element(element, kmem_cache_size(pool->pool_data));  in poison_element()
97 	} else if (pool->alloc == mempool_alloc_pages) {  in poison_element()
99 		int order = (int)(long)pool->pool_data;  in poison_element()
107 static inline void check_element(mempool_t *pool, void *element)  in check_element()  argument
110 static inline void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
115 static __always_inline bool kasan_poison_element(mempool_t *pool, void *element)  in kasan_poison_element()  argument
117 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)  in kasan_poison_element()
119 	else if (pool->alloc == mempool_alloc_pages)  in kasan_poison_element()
121 						(unsigned long)pool->pool_data);  in kasan_poison_element()
125 static void kasan_unpoison_element(mempool_t *pool, void *element)  in kasan_unpoison_element()  argument
127 	if (pool->alloc == mempool_kmalloc)  in kasan_unpoison_element()
128 		kasan_mempool_unpoison_object(element, (size_t)pool->pool_data);  in kasan_unpoison_element()
129 	else if (pool->alloc == mempool_alloc_slab)  in kasan_unpoison_element()
131 					      kmem_cache_size(pool->pool_data));  in kasan_unpoison_element()
132 	else if (pool->alloc == mempool_alloc_pages)  in kasan_unpoison_element()
134 					     (unsigned long)pool->pool_data);  in kasan_unpoison_element()
137 static __always_inline void add_element(mempool_t *pool, void *element)  in add_element()  argument
139 	BUG_ON(pool->curr_nr >= pool->min_nr);  in add_element()
140 	poison_element(pool, element);  in add_element()
141 	if (kasan_poison_element(pool, element))  in add_element()
142 		pool->elements[pool->curr_nr++] = element;  in add_element()
145 static void *remove_element(mempool_t *pool)  in remove_element()  argument
147 	void *element = pool->elements[--pool->curr_nr];  in remove_element()
149 	BUG_ON(pool->curr_nr < 0);  in remove_element()
150 	kasan_unpoison_element(pool, element);  in remove_element()
151 	check_element(pool, element);  in remove_element()
166 void mempool_exit(mempool_t *pool)  in mempool_exit()  argument
168 	while (pool->curr_nr) {  in mempool_exit()
169 		void *element = remove_element(pool);  in mempool_exit()
170 		pool->free(element, pool->pool_data);  in mempool_exit()
172 	kfree(pool->elements);  in mempool_exit()
173 	pool->elements = NULL;  in mempool_exit()
185 void mempool_destroy(mempool_t *pool)  in mempool_destroy()  argument
187 	if (unlikely(!pool))  in mempool_destroy()
190 	mempool_exit(pool);  in mempool_destroy()
191 	kfree(pool);  in mempool_destroy()
195 int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,  in mempool_init_node()  argument
199 	spin_lock_init(&pool->lock);  in mempool_init_node()
200 	pool->min_nr	= min_nr;  in mempool_init_node()
201 	pool->pool_data = pool_data;  in mempool_init_node()
202 	pool->alloc	= alloc_fn;  in mempool_init_node()
203 	pool->free	= free_fn;  in mempool_init_node()
204 	init_waitqueue_head(&pool->wait);  in mempool_init_node()
206 	pool->elements = kmalloc_array_node(min_nr, sizeof(void *),  in mempool_init_node()
208 	if (!pool->elements)  in mempool_init_node()
214 	while (pool->curr_nr < pool->min_nr) {  in mempool_init_node()
217 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_init_node()
219 			mempool_exit(pool);  in mempool_init_node()
222 		add_element(pool, element);  in mempool_init_node()
243 int mempool_init_noprof(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,  in mempool_init_noprof()  argument
246 	return mempool_init_node(pool, min_nr, alloc_fn, free_fn,  in mempool_init_noprof()
274 	mempool_t *pool;  in mempool_create_node_noprof()  local
276 	pool = kmalloc_node_noprof(sizeof(*pool), gfp_mask | __GFP_ZERO, node_id);  in mempool_create_node_noprof()
277 	if (!pool)  in mempool_create_node_noprof()
280 	if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,  in mempool_create_node_noprof()
282 		kfree(pool);  in mempool_create_node_noprof()
286 	return pool;  in mempool_create_node_noprof()
308 int mempool_resize(mempool_t *pool, int new_min_nr)  in mempool_resize()  argument
317 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
318 	if (new_min_nr <= pool->min_nr) {  in mempool_resize()
319 		while (new_min_nr < pool->curr_nr) {  in mempool_resize()
320 			element = remove_element(pool);  in mempool_resize()
321 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
322 			pool->free(element, pool->pool_data);  in mempool_resize()
323 			spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
325 		pool->min_nr = new_min_nr;  in mempool_resize()
328 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
336 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
337 	if (unlikely(new_min_nr <= pool->min_nr)) {  in mempool_resize()
339 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
343 	memcpy(new_elements, pool->elements,  in mempool_resize()
344 			pool->curr_nr * sizeof(*new_elements));  in mempool_resize()
345 	kfree(pool->elements);  in mempool_resize()
346 	pool->elements = new_elements;  in mempool_resize()
347 	pool->min_nr = new_min_nr;  in mempool_resize()
349 	while (pool->curr_nr < pool->min_nr) {  in mempool_resize()
350 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
351 		element = pool->alloc(GFP_KERNEL, pool->pool_data);  in mempool_resize()
354 		spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
355 		if (pool->curr_nr < pool->min_nr) {  in mempool_resize()
356 			add_element(pool, element);  in mempool_resize()
358 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
359 			pool->free(element, pool->pool_data);	/* Raced */  in mempool_resize()
364 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
384 void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask)  in mempool_alloc_noprof()  argument
402 	element = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc_noprof()
406 	spin_lock_irqsave(&pool->lock, flags);  in mempool_alloc_noprof()
407 	if (likely(pool->curr_nr)) {  in mempool_alloc_noprof()
408 		element = remove_element(pool);  in mempool_alloc_noprof()
409 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc_noprof()
425 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc_noprof()
432 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc_noprof()
438 	prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);  in mempool_alloc_noprof()
440 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc_noprof()
448 	finish_wait(&pool->wait, &wait);  in mempool_alloc_noprof()
466 void *mempool_alloc_preallocated(mempool_t *pool)  in mempool_alloc_preallocated()  argument
471 	spin_lock_irqsave(&pool->lock, flags);  in mempool_alloc_preallocated()
472 	if (likely(pool->curr_nr)) {  in mempool_alloc_preallocated()
473 		element = remove_element(pool);  in mempool_alloc_preallocated()
474 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc_preallocated()
484 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc_preallocated()
498 void mempool_free(void *element, mempool_t *pool)  in mempool_free()  argument
538 	if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {  in mempool_free()
539 		spin_lock_irqsave(&pool->lock, flags);  in mempool_free()
540 		if (likely(pool->curr_nr < pool->min_nr)) {  in mempool_free()
541 			add_element(pool, element);  in mempool_free()
542 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
543 			wake_up(&pool->wait);  in mempool_free()
546 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
548 	pool->free(element, pool->pool_data);  in mempool_free()