1 // SPDX-License-Identifier: GPL-2.0-only
9  * Documentation/dev-tools/kmemleak.rst.
12  * ----------------
16  * - kmemleak_lock (raw_spinlock_t): protects the object_list as well as
21  *   red black trees used to look-up metadata based on a pointer to the
26  * - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
33  * - scan_mutex (mutex): ensures that only one thread may scan the memory for
46  *   scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
48  * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
125 #define KMEMLEAK_BLACK	-1
130  * object->lock. Insertions or deletions from object_list, gray_list or
132  * the notes on locking above). These objects are reference-counted
149 	/* minimum number of a pointers found before it is considered leak */
159 	pid_t pid;			/* pid of the current task */
173 /* flag set for per-CPU pointers */
193 /* the list of gray-colored objects (see color_gray comment below) */
227 /* minimum and maximum address that may be valid per-CPU pointers */
293  * with the object->lock held.
298 	const u8 *ptr = (const u8 *)object->pointer;  in hex_dump_object()
301 	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))  in hex_dump_object()
304 	if (object->flags & OBJECT_PERCPU)  in hex_dump_object()
305 		ptr = (const u8 *)this_cpu_ptr((void __percpu *)object->pointer);  in hex_dump_object()
308 	len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);  in hex_dump_object()
310 	if (object->flags & OBJECT_PERCPU)  in hex_dump_object()
323  * - white - orphan object, not enough references to it (count < min_count)
324  * - gray  - not orphan, not marked as false positive (min_count == 0) or
326  * - black - ignore, it doesn't contain references (e.g. text section)
327  *		(min_count == -1). No function defined for this color.
328  * Newly created objects don't have any color assigned (object->count == -1)
333 	return object->count != KMEMLEAK_BLACK &&  in color_white()
334 		object->count < object->min_count;  in color_white()
339 	return object->min_count != KMEMLEAK_BLACK &&  in color_gray()
340 		object->count >= object->min_count;  in color_gray()
350 	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&  in unreferenced_object()
351 		time_before_eq(object->jiffies + jiffies_min_age,  in unreferenced_object()
357  * print_unreferenced function must be called with the object->lock held.
362 	int i;  in print_unreferenced()  local
366 	nr_entries = stack_depot_fetch(object->trace_handle, &entries);  in print_unreferenced()
368 			  object->pointer, object->size);  in print_unreferenced()
370 			   object->comm, object->pid, object->jiffies);  in print_unreferenced()
372 	warn_or_seq_printf(seq, "  backtrace (crc %x):\n", object->checksum);  in print_unreferenced()
374 	for (i = 0; i < nr_entries; i++) {  in print_unreferenced()
375 		void *ptr = (void *)entries[i];  in print_unreferenced()
383  * the object->lock held.
388 			object->pointer, object->size);  in dump_object_info()
390 			object->comm, object->pid, object->jiffies);  in dump_object_info()
391 	pr_notice("  min_count = %d\n", object->min_count);  in dump_object_info()
392 	pr_notice("  count = %d\n", object->count);  in dump_object_info()
393 	pr_notice("  flags = 0x%x\n", object->flags);  in dump_object_info()
394 	pr_notice("  checksum = %u\n", object->checksum);  in dump_object_info()
396 	if (object->trace_handle)  in dump_object_info()
397 		stack_depot_print(object->trace_handle);  in dump_object_info()
410  * Look-up a memory block metadata (kmemleak_object) in the object search
418 	struct rb_node *rb = object_tree(objflags)->rb_node;  in __lookup_object()
426 		untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);  in __lookup_object()
429 			rb = object->rb_node.rb_left;  in __lookup_object()
430 		else if (untagged_objp + object->size <= untagged_ptr)  in __lookup_object()
431 			rb = object->rb_node.rb_right;  in __lookup_object()
444 /* Look-up a kmemleak object which allocated with virtual address. */
458 	return atomic_inc_not_zero(&object->use_count);  in get_object()
482 		list_del(&object->object_list);  in mem_pool_alloc()
484 		object = &mem_pool[--mem_pool_free_count];  in mem_pool_alloc()
506 	list_add(&object->object_list, &mem_pool_free_list);  in mem_pool_free()
524 	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {  in free_object_rcu()
525 		hlist_del(&area->node);  in free_object_rcu()
533  * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
535  * recursive call to the kernel allocator. Lock-less RCU object_list traversal
540 	if (!atomic_dec_and_test(&object->use_count))  in put_object()
544 	WARN_ON(object->flags & OBJECT_ALLOCATED);  in put_object()
552 		call_rcu(&object->rcu, free_object_rcu);  in put_object()
554 		free_object_rcu(&object->rcu);  in put_object()
591 	rb_erase(&object->rb_node, object_tree(object->flags));  in __remove_object()
592 	if (!(object->del_state & DELSTATE_NO_DELETE))  in __remove_object()
593 		list_del_rcu(&object->object_list);  in __remove_object()
594 	object->del_state |= DELSTATE_REMOVED;  in __remove_object()
658 	INIT_LIST_HEAD(&object->object_list);  in __alloc_object()
659 	INIT_LIST_HEAD(&object->gray_list);  in __alloc_object()
660 	INIT_HLIST_HEAD(&object->area_list);  in __alloc_object()
661 	raw_spin_lock_init(&object->lock);  in __alloc_object()
662 	atomic_set(&object->use_count, 1);  in __alloc_object()
663 	object->excess_ref = 0;  in __alloc_object()
664 	object->count = 0;			/* white color initially */  in __alloc_object()
665 	object->checksum = 0;  in __alloc_object()
666 	object->del_state = 0;  in __alloc_object()
670 		object->pid = 0;  in __alloc_object()
671 		strscpy(object->comm, "hardirq");  in __alloc_object()
673 		object->pid = 0;  in __alloc_object()
674 		strscpy(object->comm, "softirq");  in __alloc_object()
676 		object->pid = current->pid;  in __alloc_object()
680 		 * dependency issues with current->alloc_lock. In the worst  in __alloc_object()
683 		strscpy(object->comm, current->comm);  in __alloc_object()
687 	object->trace_handle = set_track_prepare();  in __alloc_object()
701 	object->flags = OBJECT_ALLOCATED | objflags;  in __link_object()
702 	object->pointer = ptr;  in __link_object()
703 	object->size = kfence_ksize((void *)ptr) ?: size;  in __link_object()
704 	object->min_count = min_count;  in __link_object()
705 	object->jiffies = jiffies;  in __link_object()
710 	 * address. And update min_percpu_addr max_percpu_addr for per-CPU  in __link_object()
720 	link = &object_tree(objflags)->rb_node;  in __link_object()
725 		untagged_objp = (unsigned long)kasan_reset_tag((void *)parent->pointer);  in __link_object()
727 			link = &parent->rb_node.rb_left;  in __link_object()
728 		else if (untagged_objp + parent->size <= untagged_ptr)  in __link_object()
729 			link = &parent->rb_node.rb_right;  in __link_object()
734 			 * No need for parent->lock here since "parent" cannot  in __link_object()
738 			return -EEXIST;  in __link_object()
741 	rb_link_node(&object->rb_node, rb_parent, link);  in __link_object()
742 	rb_insert_color(&object->rb_node, object_tree(objflags));  in __link_object()
743 	list_add_tail_rcu(&object->object_list, &object_list);  in __link_object()
784 /* Create kmemleak object corresponding to a per-CPU allocation. */
798 	WARN_ON(!(object->flags & OBJECT_ALLOCATED));  in __delete_object()
799 	WARN_ON(atomic_read(&object->use_count) < 1);  in __delete_object()
805 	raw_spin_lock_irqsave(&object->lock, flags);  in __delete_object()
806 	object->flags &= ~OBJECT_ALLOCATED;  in __delete_object()
807 	raw_spin_unlock_irqrestore(&object->lock, flags);  in __delete_object()
864 	start = object->pointer;  in delete_object_part()
865 	end = object->pointer + object->size;  in delete_object_part()
867 	    !__link_object(object_l, start, ptr - start,  in delete_object_part()
868 			   object->min_count, objflags))  in delete_object_part()
871 	    !__link_object(object_r, ptr + size, end - ptr - size,  in delete_object_part()
872 			   object->min_count, objflags))  in delete_object_part()
889 	object->min_count = color;  in __paint_it()
891 		object->flags |= OBJECT_NO_SCAN;  in __paint_it()
898 	raw_spin_lock_irqsave(&object->lock, flags);  in paint_it()
900 	raw_spin_unlock_irqrestore(&object->lock, flags);  in paint_it()
920  * Mark an object permanently as gray-colored so that it can no longer be
921  * reported as a leak. This is used in general to mark a false positive.
929  * Mark the object as black-colored so that it is ignored from scans and
957 	untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);  in add_scan_area()
963 	raw_spin_lock_irqsave(&object->lock, flags);  in add_scan_area()
967 		object->flags |= OBJECT_FULL_SCAN;  in add_scan_area()
971 		size = untagged_objp + object->size - untagged_ptr;  in add_scan_area()
972 	} else if (untagged_ptr + size > untagged_objp + object->size) {  in add_scan_area()
979 	INIT_HLIST_NODE(&area->node);  in add_scan_area()
980 	area->start = ptr;  in add_scan_area()
981 	area->size = size;  in add_scan_area()
983 	hlist_add_head(&area->node, &object->area_list);  in add_scan_area()
985 	raw_spin_unlock_irqrestore(&object->lock, flags);  in add_scan_area()
1007 	raw_spin_lock_irqsave(&object->lock, flags);  in object_set_excess_ref()
1008 	object->excess_ref = excess_ref;  in object_set_excess_ref()
1009 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_set_excess_ref()
1029 	raw_spin_lock_irqsave(&object->lock, flags);  in object_no_scan()
1030 	object->flags |= OBJECT_NO_SCAN;  in object_no_scan()
1031 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_no_scan()
1036  * kmemleak_alloc - register a newly allocated object
1041  *		the object is reported as a memory leak. If @min_count is 0,
1042  *		the object is never reported as a leak. If @min_count is -1,
1043  *		the object is ignored (not scanned and not reported as a leak)
1060  * kmemleak_alloc_percpu - register a newly allocated __percpu object
1079  * kmemleak_vmalloc - register a newly vmalloc'ed object
1096 		create_object((unsigned long)area->addr, size, 2, gfp);  in kmemleak_vmalloc()
1098 				      (unsigned long)area->addr);  in kmemleak_vmalloc()
1104  * kmemleak_free - unregister a previously registered object
1120  * kmemleak_free_part - partially unregister a previously registered object
1138  * kmemleak_free_percpu - unregister a previously registered __percpu object
1154  * kmemleak_update_trace - update object allocation stack trace
1181 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_update_trace()
1182 	object->trace_handle = trace_handle;  in kmemleak_update_trace()
1183 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_update_trace()
1190  * kmemleak_not_leak - mark an allocated object as false positive
1194  * be reported as leak and always be scanned.
1206  * kmemleak_ignore - ignore an allocated object
1210  * ignored (not scanned and not reported as a leak). This is usually done when
1211  * it is known that the corresponding block is not a leak and does not contain
1224  * kmemleak_scan_area - limit the range to be scanned in an allocated object
1244  * kmemleak_no_scan - do not scan an allocated object
1262  * kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1282  * kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1298  * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1316 	u32 old_csum = object->checksum;  in update_checksum()
1318 	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))  in update_checksum()
1323 	if (object->flags & OBJECT_PERCPU) {  in update_checksum()
1326 		object->checksum = 0;  in update_checksum()
1328 			void *ptr = per_cpu_ptr((void __percpu *)object->pointer, cpu);  in update_checksum()
1330 			object->checksum ^= crc32(0, kasan_reset_tag((void *)ptr), object->size);  in update_checksum()
1333 		object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);  in update_checksum()
1338 	return object->checksum != old_csum;  in update_checksum()
1342  * Update an object's references. object->lock must be held by the caller.
1347 		/* non-orphan, ignored or new */  in update_refs()
1357 	object->count++;  in update_refs()
1361 		list_add_tail(&object->gray_list, &gray_list);  in update_refs()
1383 	 * object->use_count cannot be dropped to 0 while the object  in pointer_update_refs()
1395 	 * Avoid the lockdep recursive warning on object->lock being  in pointer_update_refs()
1399 	raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in pointer_update_refs()
1402 		excess_ref = object->excess_ref;  in pointer_update_refs()
1408 	raw_spin_unlock(&object->lock);  in pointer_update_refs()
1417 		raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in pointer_update_refs()
1419 		raw_spin_unlock(&object->lock);  in pointer_update_refs()
1436 	if (current->mm)  in scan_should_stop()
1437 		return signal_pending(current);  in scan_should_stop()
1453 	unsigned long *end = _end - (BYTES_PER_POINTER - 1);  in scan_block()
1492  * that object->use_count >= 1.
1500 	 * Once the object->lock is acquired, the corresponding memory block  in scan_object()
1503 	raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1504 	if (object->flags & OBJECT_NO_SCAN)  in scan_object()
1506 	if (!(object->flags & OBJECT_ALLOCATED))  in scan_object()
1510 	if (object->flags & OBJECT_PERCPU) {  in scan_object()
1514 			void *start = per_cpu_ptr((void __percpu *)object->pointer, cpu);  in scan_object()
1515 			void *end = start + object->size;  in scan_object()
1519 			raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1521 			raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1522 			if (!(object->flags & OBJECT_ALLOCATED))  in scan_object()
1525 	} else if (hlist_empty(&object->area_list) ||  in scan_object()
1526 	    object->flags & OBJECT_FULL_SCAN) {  in scan_object()
1527 		void *start = object->flags & OBJECT_PHYS ?  in scan_object()
1528 				__va((phys_addr_t)object->pointer) :  in scan_object()
1529 				(void *)object->pointer;  in scan_object()
1530 		void *end = start + object->size;  in scan_object()
1541 			raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1543 			raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1544 		} while (object->flags & OBJECT_ALLOCATED);  in scan_object()
1546 		hlist_for_each_entry(area, &object->area_list, node)  in scan_object()
1547 			scan_block((void *)area->start,  in scan_object()
1548 				   (void *)(area->start + area->size),  in scan_object()
1552 	raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1569 	while (&object->gray_list != &gray_list) {  in scan_gray_list()
1576 		tmp = list_entry(object->gray_list.next, typeof(*object),  in scan_gray_list()
1580 		list_del(&object->gray_list);  in scan_gray_list()
1599 	if (object->del_state & DELSTATE_REMOVED)  in kmemleak_cond_resched()
1601 	object->del_state |= DELSTATE_NO_DELETE;  in kmemleak_cond_resched()
1609 	if (object->del_state & DELSTATE_REMOVED)  in kmemleak_cond_resched()
1610 		list_del_rcu(&object->object_list);  in kmemleak_cond_resched()
1611 	object->del_state &= ~DELSTATE_NO_DELETE;  in kmemleak_cond_resched()
1626 	int __maybe_unused i;  in kmemleak_scan()  local
1634 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1640 		if (atomic_read(&object->use_count) > 1) {  in kmemleak_scan()
1641 			pr_debug("object->use_count = %d\n",  in kmemleak_scan()
1642 				 atomic_read(&object->use_count));  in kmemleak_scan()
1648 		if ((object->flags & OBJECT_PHYS) &&  in kmemleak_scan()
1649 		   !(object->flags & OBJECT_NO_SCAN)) {  in kmemleak_scan()
1650 			unsigned long phys = object->pointer;  in kmemleak_scan()
1653 			    PHYS_PFN(phys + object->size) >= max_low_pfn)  in kmemleak_scan()
1658 		object->count = 0;  in kmemleak_scan()
1660 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1662 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1670 	/* per-cpu sections scanning */  in kmemleak_scan()
1671 	for_each_possible_cpu(i)  in kmemleak_scan()
1672 		scan_large_block(__per_cpu_start + per_cpu_offset(i),  in kmemleak_scan()
1673 				 __per_cpu_end + per_cpu_offset(i));  in kmemleak_scan()
1681 		unsigned long start_pfn = zone->zone_start_pfn;  in kmemleak_scan()
1744 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1745 		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)  in kmemleak_scan()
1748 			object->count = object->min_count;  in kmemleak_scan()
1749 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1751 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1756 	 * Re-scan the gray list for modified unreferenced objects.  in kmemleak_scan()
1781 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1783 		    !(object->flags & OBJECT_REPORTED)) {  in kmemleak_scan()
1784 			object->flags |= OBJECT_REPORTED;  in kmemleak_scan()
1791 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1813 	set_user_nice(current, 10);  in kmemleak_scan_thread()
1885 		if (n-- > 0)  in kmemleak_seq_start()
1943 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_seq_show()
1944 	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))  in kmemleak_seq_show()
1946 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_seq_show()
1969 		return -EINVAL;  in dump_str_object_info()
1973 		return -EINVAL;  in dump_str_object_info()
1976 	raw_spin_lock_irqsave(&object->lock, flags);  in dump_str_object_info()
1978 	raw_spin_unlock_irqrestore(&object->lock, flags);  in dump_str_object_info()
1996 		raw_spin_lock_irq(&object->lock);  in kmemleak_clear()
1997 		if ((object->flags & OBJECT_REPORTED) &&  in kmemleak_clear()
2000 		raw_spin_unlock_irq(&object->lock);  in kmemleak_clear()
2010  * File write operation to configure kmemleak at run-time. The following
2012  *   off	- disable kmemleak (irreversible)
2013  *   stack=on	- enable the task stacks scanning
2014  *   stack=off	- disable the tasks stacks scanning
2015  *   scan=on	- start the automatic memory scanning thread
2016  *   scan=off	- stop the automatic memory scanning thread
2017  *   scan=...	- set the automatic memory scanning period in seconds (0 to
2019  *   scan	- trigger a memory scan
2020  *   clear	- mark all current reported unreferenced kmemleak objects as
2023  *   dump=...	- dump information about the object found at the given address
2032 	buf_size = min(size, (sizeof(buf) - 1));  in kmemleak_write()
2034 		return -EFAULT;  in kmemleak_write()
2050 		ret = -EPERM;  in kmemleak_write()
2086 		ret = -EINVAL;  in kmemleak_write()
2167 	pr_info("Kernel memory leak detector disabled\n");  in kmemleak_disable()
2171  * Allow boot-time kmemleak disabling (enabled by default).
2176 		return -EINVAL;  in kmemleak_boot_config()
2184 		return -EINVAL;  in kmemleak_boot_config()
2211 	create_object((unsigned long)_sdata, _edata - _sdata,  in kmemleak_init()
2213 	create_object((unsigned long)__bss_start, __bss_stop - __bss_start,  in kmemleak_init()
2218 			      __end_ro_after_init - __start_ro_after_init,  in kmemleak_init()
2236 		 * two clean-up threads but serialized by scan_mutex.  in kmemleak_late_init()
2239 		return -ENOMEM;  in kmemleak_late_init()
2248 	pr_info("Kernel memory leak detector initialized (mem pool available: %d)\n",  in kmemleak_late_init()