7  * DOC: VC4 GEM BO management support
9  * The VC4 GPU architecture (both scanout and rendering) has direct
44 static void vc4_bo_stats_print(struct drm_printer *p, struct vc4_dev *vc4)  in vc4_bo_stats_print()  argument
48 	for (i = 0; i < vc4->num_labels; i++) {  in vc4_bo_stats_print()
49 		if (!vc4->bo_labels[i].num_allocated)  in vc4_bo_stats_print()
53 			   vc4->bo_labels[i].name,  in vc4_bo_stats_print()
54 			   vc4->bo_labels[i].size_allocated / 1024,  in vc4_bo_stats_print()
55 			   vc4->bo_labels[i].num_allocated);  in vc4_bo_stats_print()
58 	mutex_lock(&vc4->purgeable.lock);  in vc4_bo_stats_print()
59 	if (vc4->purgeable.num)  in vc4_bo_stats_print()
61 			   vc4->purgeable.size / 1024, vc4->purgeable.num);  in vc4_bo_stats_print()
63 	if (vc4->purgeable.purged_num)  in vc4_bo_stats_print()
65 			   vc4->purgeable.purged_size / 1024,  in vc4_bo_stats_print()
66 			   vc4->purgeable.purged_num);  in vc4_bo_stats_print()
67 	mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_stats_print()
74 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_stats_debugfs()  local
77 	vc4_bo_stats_print(&p, vc4);  in vc4_bo_stats_debugfs()
90 static int vc4_get_user_label(struct vc4_dev *vc4, const char *name)  in vc4_get_user_label()  argument
95 	for (i = 0; i < vc4->num_labels; i++) {  in vc4_get_user_label()
96 		if (!vc4->bo_labels[i].name) {  in vc4_get_user_label()
98 		} else if (strcmp(vc4->bo_labels[i].name, name) == 0) {  in vc4_get_user_label()
105 		WARN_ON(vc4->bo_labels[free_slot].num_allocated != 0);  in vc4_get_user_label()
106 		vc4->bo_labels[free_slot].name = name;  in vc4_get_user_label()
109 		u32 new_label_count = vc4->num_labels + 1;  in vc4_get_user_label()
111 			krealloc(vc4->bo_labels,  in vc4_get_user_label()
120 		free_slot = vc4->num_labels;  in vc4_get_user_label()
121 		vc4->bo_labels = new_labels;  in vc4_get_user_label()
122 		vc4->num_labels = new_label_count;  in vc4_get_user_label()
124 		vc4->bo_labels[free_slot].name = name;  in vc4_get_user_label()
125 		vc4->bo_labels[free_slot].num_allocated = 0;  in vc4_get_user_label()
126 		vc4->bo_labels[free_slot].size_allocated = 0;  in vc4_get_user_label()
135 	struct vc4_dev *vc4 = to_vc4_dev(gem_obj->dev);  in vc4_bo_set_label()  local
137 	lockdep_assert_held(&vc4->bo_lock);  in vc4_bo_set_label()
140 		vc4->bo_labels[label].num_allocated++;  in vc4_bo_set_label()
141 		vc4->bo_labels[label].size_allocated += gem_obj->size;  in vc4_bo_set_label()
144 	vc4->bo_labels[bo->label].num_allocated--;  in vc4_bo_set_label()
145 	vc4->bo_labels[bo->label].size_allocated -= gem_obj->size;  in vc4_bo_set_label()
147 	if (vc4->bo_labels[bo->label].num_allocated == 0 &&  in vc4_bo_set_label()
154 		kfree(vc4->bo_labels[bo->label].name);  in vc4_bo_set_label()
155 		vc4->bo_labels[bo->label].name = NULL;  in vc4_bo_set_label()
169 	struct vc4_dev *vc4 = to_vc4_dev(obj->dev);  in vc4_bo_destroy()  local
171 	lockdep_assert_held(&vc4->bo_lock);  in vc4_bo_destroy()
188 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_remove_from_cache()  local
190 	lockdep_assert_held(&vc4->bo_lock);  in vc4_bo_remove_from_cache()
198 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_get_cache_list_for_size()  local
201 	if (vc4->bo_cache.size_list_size <= page_index) {  in vc4_get_cache_list_for_size()
202 		uint32_t new_size = max(vc4->bo_cache.size_list_size * 2,  in vc4_get_cache_list_for_size()
215 		for (i = 0; i < vc4->bo_cache.size_list_size; i++) {  in vc4_get_cache_list_for_size()
217 				&vc4->bo_cache.size_list[i];  in vc4_get_cache_list_for_size()
225 		for (i = vc4->bo_cache.size_list_size; i < new_size; i++)  in vc4_get_cache_list_for_size()
228 		kfree(vc4->bo_cache.size_list);  in vc4_get_cache_list_for_size()
229 		vc4->bo_cache.size_list = new_list;  in vc4_get_cache_list_for_size()
230 		vc4->bo_cache.size_list_size = new_size;  in vc4_get_cache_list_for_size()
233 	return &vc4->bo_cache.size_list[page_index];  in vc4_get_cache_list_for_size()
238 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_cache_purge()  local
240 	mutex_lock(&vc4->bo_lock);  in vc4_bo_cache_purge()
241 	while (!list_empty(&vc4->bo_cache.time_list)) {  in vc4_bo_cache_purge()
242 		struct vc4_bo *bo = list_last_entry(&vc4->bo_cache.time_list,  in vc4_bo_cache_purge()
247 	mutex_unlock(&vc4->bo_lock);  in vc4_bo_cache_purge()
252 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_add_to_purgeable_pool()  local
254 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_add_to_purgeable_pool()
257 	mutex_lock(&vc4->purgeable.lock);  in vc4_bo_add_to_purgeable_pool()
258 	list_add_tail(&bo->size_head, &vc4->purgeable.list);  in vc4_bo_add_to_purgeable_pool()
259 	vc4->purgeable.num++;  in vc4_bo_add_to_purgeable_pool()
260 	vc4->purgeable.size += bo->base.base.size;  in vc4_bo_add_to_purgeable_pool()
261 	mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_add_to_purgeable_pool()
266 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_remove_from_purgeable_pool_locked()  local
268 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_remove_from_purgeable_pool_locked()
284 	vc4->purgeable.num--;  in vc4_bo_remove_from_purgeable_pool_locked()
285 	vc4->purgeable.size -= bo->base.base.size;  in vc4_bo_remove_from_purgeable_pool_locked()
290 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_remove_from_purgeable_pool()  local
292 	mutex_lock(&vc4->purgeable.lock);  in vc4_bo_remove_from_purgeable_pool()
294 	mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_remove_from_purgeable_pool()
314 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_userspace_cache_purge()  local
316 	mutex_lock(&vc4->purgeable.lock);  in vc4_bo_userspace_cache_purge()
317 	while (!list_empty(&vc4->purgeable.list)) {  in vc4_bo_userspace_cache_purge()
318 		struct vc4_bo *bo = list_first_entry(&vc4->purgeable.list,  in vc4_bo_userspace_cache_purge()
330 		mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_userspace_cache_purge()
349 		mutex_lock(&vc4->purgeable.lock);  in vc4_bo_userspace_cache_purge()
352 			vc4->purgeable.purged_size += purged_size;  in vc4_bo_userspace_cache_purge()
353 			vc4->purgeable.purged_num++;  in vc4_bo_userspace_cache_purge()
356 	mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_userspace_cache_purge()
363 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_get_from_cache()  local
367 	mutex_lock(&vc4->bo_lock);  in vc4_bo_get_from_cache()
368 	if (page_index >= vc4->bo_cache.size_list_size)  in vc4_bo_get_from_cache()
371 	if (list_empty(&vc4->bo_cache.size_list[page_index]))  in vc4_bo_get_from_cache()
374 	bo = list_first_entry(&vc4->bo_cache.size_list[page_index],  in vc4_bo_get_from_cache()
382 	mutex_unlock(&vc4->bo_lock);  in vc4_bo_get_from_cache()
396 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_create_object()  local
399 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_create_object()
411 	mutex_lock(&vc4->bo_lock);  in vc4_create_object()
413 	vc4->bo_labels[VC4_BO_TYPE_KERNEL].num_allocated++;  in vc4_create_object()
414 	vc4->bo_labels[VC4_BO_TYPE_KERNEL].size_allocated += size;  in vc4_create_object()
415 	mutex_unlock(&vc4->bo_lock);  in vc4_create_object()
426 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_create()  local
430 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_create()
471 		struct drm_printer p = drm_info_printer(vc4->base.dev);  in vc4_bo_create()
473 		vc4_bo_stats_print(&p, vc4);  in vc4_bo_create()
484 	mutex_lock(&vc4->bo_lock);  in vc4_bo_create()
486 	mutex_unlock(&vc4->bo_lock);  in vc4_bo_create()
495 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_dumb_create()  local
499 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_dumb_create()
520 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_cache_free_old()  local
523 	lockdep_assert_held(&vc4->bo_lock);  in vc4_bo_cache_free_old()
525 	while (!list_empty(&vc4->bo_cache.time_list)) {  in vc4_bo_cache_free_old()
526 		struct vc4_bo *bo = list_last_entry(&vc4->bo_cache.time_list,  in vc4_bo_cache_free_old()
529 			mod_timer(&vc4->bo_cache.time_timer,  in vc4_bo_cache_free_old()
546 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_free_object()  local
556 	mutex_lock(&vc4->bo_lock);  in vc4_free_object()
599 	list_add(&bo->unref_head, &vc4->bo_cache.time_list);  in vc4_free_object()
606 	mutex_unlock(&vc4->bo_lock);  in vc4_free_object()
611 	struct vc4_dev *vc4 =  in vc4_bo_cache_time_work()  local
613 	struct drm_device *dev = &vc4->base;  in vc4_bo_cache_time_work()
615 	mutex_lock(&vc4->bo_lock);  in vc4_bo_cache_time_work()
617 	mutex_unlock(&vc4->bo_lock);  in vc4_bo_cache_time_work()
622 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_inc_usecnt()  local
625 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_inc_usecnt()
662 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_dec_usecnt()  local
664 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_dec_usecnt()
682 	struct vc4_dev *vc4 = from_timer(vc4, t, bo_cache.time_timer);  in vc4_bo_cache_time_timer()  local
684 	schedule_work(&vc4->bo_cache.time_work);  in vc4_bo_cache_time_timer()
766 static int vc4_grab_bin_bo(struct vc4_dev *vc4, struct vc4_file *vc4file)  in vc4_grab_bin_bo()  argument
768 	if (!vc4->v3d)  in vc4_grab_bin_bo()
774 	return vc4_v3d_bin_bo_get(vc4, &vc4file->bin_bo_used);  in vc4_grab_bin_bo()
782 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_create_bo_ioctl()  local
786 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_create_bo_ioctl()
789 	ret = vc4_grab_bin_bo(vc4, vc4file);  in vc4_create_bo_ioctl()
812 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_mmap_bo_ioctl()  local
816 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_mmap_bo_ioctl()
838 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_create_shader_bo_ioctl()  local
842 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_create_shader_bo_ioctl()
861 	ret = vc4_grab_bin_bo(vc4, vc4file);  in vc4_create_shader_bo_ioctl()
915 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_set_tiling_ioctl()  local
921 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_set_tiling_ioctl()
962 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_get_tiling_ioctl()  local
967 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_get_tiling_ioctl()
993 	struct vc4_dev *vc4 = to_vc4_dev(drm);  in vc4_bo_debugfs_init()  local
995 	if (!vc4->v3d)  in vc4_bo_debugfs_init()
1006 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_cache_init()  local
1010 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_cache_init()
1017 	vc4->bo_labels = kcalloc(VC4_BO_TYPE_COUNT, sizeof(*vc4->bo_labels),  in vc4_bo_cache_init()
1019 	if (!vc4->bo_labels)  in vc4_bo_cache_init()
1021 	vc4->num_labels = VC4_BO_TYPE_COUNT;  in vc4_bo_cache_init()
1025 		vc4->bo_labels[i].name = bo_type_names[i];  in vc4_bo_cache_init()
1027 	ret = drmm_mutex_init(dev, &vc4->bo_lock);  in vc4_bo_cache_init()
1029 		kfree(vc4->bo_labels);  in vc4_bo_cache_init()
1033 	INIT_LIST_HEAD(&vc4->bo_cache.time_list);  in vc4_bo_cache_init()
1035 	INIT_WORK(&vc4->bo_cache.time_work, vc4_bo_cache_time_work);  in vc4_bo_cache_init()
1036 	timer_setup(&vc4->bo_cache.time_timer, vc4_bo_cache_time_timer, 0);  in vc4_bo_cache_init()
1043 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_cache_destroy()  local
1046 	del_timer(&vc4->bo_cache.time_timer);  in vc4_bo_cache_destroy()
1047 	cancel_work_sync(&vc4->bo_cache.time_work);  in vc4_bo_cache_destroy()
1051 	for (i = 0; i < vc4->num_labels; i++) {  in vc4_bo_cache_destroy()
1052 		if (vc4->bo_labels[i].num_allocated) {  in vc4_bo_cache_destroy()
1055 				vc4->bo_labels[i].num_allocated,  in vc4_bo_cache_destroy()
1056 				vc4->bo_labels[i].name);  in vc4_bo_cache_destroy()
1060 			kfree(vc4->bo_labels[i].name);  in vc4_bo_cache_destroy()
1062 	kfree(vc4->bo_labels);  in vc4_bo_cache_destroy()
1068 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_label_bo_ioctl()  local
1074 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_label_bo_ioctl()
1091 	mutex_lock(&vc4->bo_lock);  in vc4_label_bo_ioctl()
1092 	label = vc4_get_user_label(vc4, name);  in vc4_label_bo_ioctl()
1097 	mutex_unlock(&vc4->bo_lock);  in vc4_label_bo_ioctl()