78 		struct bpf_cgroup_storage *storage;  in cgroup_storage_lookup()  local
80 		storage = container_of(node, struct bpf_cgroup_storage, node);  in cgroup_storage_lookup()
82 		switch (bpf_cgroup_storage_key_cmp(map, key, &storage->key)) {  in cgroup_storage_lookup()
92 			return storage;  in cgroup_storage_lookup()
103 				 struct bpf_cgroup_storage *storage)  in cgroup_storage_insert()  argument
114 		switch (bpf_cgroup_storage_key_cmp(map, &storage->key, &this->key)) {  in cgroup_storage_insert()
126 	rb_link_node(&storage->node, parent, new);  in cgroup_storage_insert()
127 	rb_insert_color(&storage->node, root);  in cgroup_storage_insert()
135 	struct bpf_cgroup_storage *storage;  in cgroup_storage_lookup_elem()  local
137 	storage = cgroup_storage_lookup(map, key, false);  in cgroup_storage_lookup_elem()
138 	if (!storage)  in cgroup_storage_lookup_elem()
141 	return &READ_ONCE(storage->buf)->data[0];  in cgroup_storage_lookup_elem()
147 	struct bpf_cgroup_storage *storage;  in cgroup_storage_update_elem()  local
157 	storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map,  in cgroup_storage_update_elem()
159 	if (!storage)  in cgroup_storage_update_elem()
163 		copy_map_value_locked(map, storage->buf->data, value, false);  in cgroup_storage_update_elem()
176 	new = xchg(&storage->buf, new);  in cgroup_storage_update_elem()
186 	struct bpf_cgroup_storage *storage;  in bpf_percpu_cgroup_storage_copy()  local
191 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_copy()
192 	if (!storage) {  in bpf_percpu_cgroup_storage_copy()
204 				per_cpu_ptr(storage->percpu_buf, cpu), size);  in bpf_percpu_cgroup_storage_copy()
215 	struct bpf_cgroup_storage *storage;  in bpf_percpu_cgroup_storage_update()  local
223 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_update()
224 	if (!storage) {  in bpf_percpu_cgroup_storage_update()
237 		bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu),  in bpf_percpu_cgroup_storage_update()
249 	struct bpf_cgroup_storage *storage;  in cgroup_storage_get_next_key()  local
257 		storage = cgroup_storage_lookup(map, key, true);  in cgroup_storage_get_next_key()
258 		if (!storage)  in cgroup_storage_get_next_key()
261 		storage = list_next_entry(storage, list_map);  in cgroup_storage_get_next_key()
262 		if (!storage)  in cgroup_storage_get_next_key()
265 		storage = list_first_entry(&map->list,  in cgroup_storage_get_next_key()
273 		*next = storage->key;  in cgroup_storage_get_next_key()
276 		*next = storage->key.cgroup_inode_id;  in cgroup_storage_get_next_key()
334 	struct bpf_cgroup_storage *storage, *stmp;  in cgroup_storage_map_free()  local
338 	list_for_each_entry_safe(storage, stmp, storages, list_map) {  in cgroup_storage_map_free()
339 		bpf_cgroup_storage_unlink(storage);  in cgroup_storage_map_free()
340 		bpf_cgroup_storage_free(storage);  in cgroup_storage_map_free()
418 	struct bpf_cgroup_storage *storage;  in cgroup_storage_seq_show_elem()  local
422 	storage = cgroup_storage_lookup(map_to_storage(map), key, false);  in cgroup_storage_seq_show_elem()
423 	if (!storage) {  in cgroup_storage_seq_show_elem()
433 				  &READ_ONCE(storage->buf)->data[0], m);  in cgroup_storage_seq_show_elem()
440 					  per_cpu_ptr(storage->percpu_buf, cpu),  in cgroup_storage_seq_show_elem()
503 	struct bpf_cgroup_storage *storage;  in bpf_cgroup_storage_alloc()  local
514 	storage = bpf_map_kmalloc_node(map, sizeof(struct bpf_cgroup_storage),  in bpf_cgroup_storage_alloc()
516 	if (!storage)  in bpf_cgroup_storage_alloc()
520 		storage->buf = bpf_map_kmalloc_node(map, size, gfp,  in bpf_cgroup_storage_alloc()
522 		if (!storage->buf)  in bpf_cgroup_storage_alloc()
524 		check_and_init_map_value(map, storage->buf->data);  in bpf_cgroup_storage_alloc()
526 		storage->percpu_buf = bpf_map_alloc_percpu(map, size, 8, gfp);  in bpf_cgroup_storage_alloc()
527 		if (!storage->percpu_buf)  in bpf_cgroup_storage_alloc()
531 	storage->map = (struct bpf_cgroup_storage_map *)map;  in bpf_cgroup_storage_alloc()
533 	return storage;  in bpf_cgroup_storage_alloc()
536 	kfree(storage);  in bpf_cgroup_storage_alloc()
542 	struct bpf_cgroup_storage *storage =  in free_shared_cgroup_storage_rcu()  local
545 	kfree(storage->buf);  in free_shared_cgroup_storage_rcu()
546 	kfree(storage);  in free_shared_cgroup_storage_rcu()
551 	struct bpf_cgroup_storage *storage =  in free_percpu_cgroup_storage_rcu()  local
554 	free_percpu(storage->percpu_buf);  in free_percpu_cgroup_storage_rcu()
555 	kfree(storage);  in free_percpu_cgroup_storage_rcu()
558 void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage)  in bpf_cgroup_storage_free()  argument
563 	if (!storage)  in bpf_cgroup_storage_free()
566 	map = &storage->map->map;  in bpf_cgroup_storage_free()
569 		call_rcu(&storage->rcu, free_shared_cgroup_storage_rcu);  in bpf_cgroup_storage_free()
571 		call_rcu(&storage->rcu, free_percpu_cgroup_storage_rcu);  in bpf_cgroup_storage_free()
574 void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,  in bpf_cgroup_storage_link()  argument
580 	if (!storage)  in bpf_cgroup_storage_link()
583 	storage->key.attach_type = type;  in bpf_cgroup_storage_link()
584 	storage->key.cgroup_inode_id = cgroup_id(cgroup);  in bpf_cgroup_storage_link()
586 	map = storage->map;  in bpf_cgroup_storage_link()
589 	WARN_ON(cgroup_storage_insert(map, storage));  in bpf_cgroup_storage_link()
590 	list_add(&storage->list_map, &map->list);  in bpf_cgroup_storage_link()
591 	list_add(&storage->list_cg, &cgroup->bpf.storages);  in bpf_cgroup_storage_link()
595 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage)  in bpf_cgroup_storage_unlink()  argument
600 	if (!storage)  in bpf_cgroup_storage_unlink()
603 	map = storage->map;  in bpf_cgroup_storage_unlink()
607 	rb_erase(&storage->node, root);  in bpf_cgroup_storage_unlink()
609 	list_del(&storage->list_map);  in bpf_cgroup_storage_unlink()
610 	list_del(&storage->list_cg);  in bpf_cgroup_storage_unlink()