104 static void bkey_cached_free(struct btree_key_cache *bc,  in bkey_cached_free()  argument
115 	rcu_pending_enqueue(&bc->pending[pcpu_readers], &ck->rcu);  in bkey_cached_free()
116 	this_cpu_inc(*bc->nr_pending);  in bkey_cached_free()
139 	struct btree_key_cache *bc = &c->btree_key_cache;  in bkey_cached_alloc()  local
144 				rcu_pending_dequeue(&bc->pending[pcpu_readers]),  in bkey_cached_alloc()
164 	ck = container_of_or_null(rcu_pending_dequeue_from_all(&bc->pending[pcpu_readers]),  in bkey_cached_alloc()
204 	struct btree_key_cache *bc = &c->btree_key_cache;  in btree_key_cache_create()  local
226 		ck = bkey_cached_reuse(bc);  in btree_key_cache_create()
261 	ret = rhashtable_lookup_insert_fast(&bc->table, &ck->hash, bch2_btree_key_cache_params);  in btree_key_cache_create()
265 	atomic_long_inc(&bc->nr_keys);  in btree_key_cache_create()
275 	bkey_cached_free(bc, ck);  in btree_key_cache_create()
572 	struct btree_key_cache *bc = &c->btree_key_cache;  in bch2_btree_key_cache_drop()  local
585 	bkey_cached_evict(bc, ck);  in bch2_btree_key_cache_drop()
586 	bkey_cached_free(bc, ck);  in bch2_btree_key_cache_drop()
597 	struct btree_key_cache *bc = &c->btree_key_cache;  in bch2_btree_key_cache_scan()  local
607 	tbl = rht_dereference_rcu(bc->table.tbl, &bc->table);  in bch2_btree_key_cache_scan()
622 	iter = bc->shrink_iter;  in bch2_btree_key_cache_scan()
637 				bc->skipped_dirty++;  in bch2_btree_key_cache_scan()
640 				bc->skipped_accessed++;  in bch2_btree_key_cache_scan()
642 				bc->skipped_lock_fail++;  in bch2_btree_key_cache_scan()
643 			} else if (bkey_cached_evict(bc, ck)) {  in bch2_btree_key_cache_scan()
644 				bkey_cached_free(bc, ck);  in bch2_btree_key_cache_scan()
645 				bc->freed++;  in bch2_btree_key_cache_scan()
664 	bc->shrink_iter = iter;  in bch2_btree_key_cache_scan()
676 	struct btree_key_cache *bc = &c->btree_key_cache;  in bch2_btree_key_cache_count()  local
677 	long nr = atomic_long_read(&bc->nr_keys) -  in bch2_btree_key_cache_count()
678 		atomic_long_read(&bc->nr_dirty);  in bch2_btree_key_cache_count()
691 void bch2_fs_btree_key_cache_exit(struct btree_key_cache *bc)  in bch2_fs_btree_key_cache_exit()  argument
693 	struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache);  in bch2_fs_btree_key_cache_exit()
700 	shrinker_free(bc->shrink);  in bch2_fs_btree_key_cache_exit()
705 	while (atomic_long_read(&bc->nr_keys)) {  in bch2_fs_btree_key_cache_exit()
707 		tbl = rht_dereference_rcu(bc->table.tbl, &bc->table);  in bch2_fs_btree_key_cache_exit()
712 				mutex_lock(&bc->table.mutex);  in bch2_fs_btree_key_cache_exit()
713 				mutex_unlock(&bc->table.mutex);  in bch2_fs_btree_key_cache_exit()
720 					BUG_ON(!bkey_cached_evict(bc, ck));  in bch2_fs_btree_key_cache_exit()
728 	if (atomic_long_read(&bc->nr_dirty) &&  in bch2_fs_btree_key_cache_exit()
732 		      atomic_long_read(&bc->nr_dirty));  in bch2_fs_btree_key_cache_exit()
734 	if (atomic_long_read(&bc->nr_keys))  in bch2_fs_btree_key_cache_exit()
736 		      atomic_long_read(&bc->nr_keys));  in bch2_fs_btree_key_cache_exit()
738 	if (bc->table_init_done)  in bch2_fs_btree_key_cache_exit()
739 		rhashtable_destroy(&bc->table);  in bch2_fs_btree_key_cache_exit()
741 	rcu_pending_exit(&bc->pending[0]);  in bch2_fs_btree_key_cache_exit()
742 	rcu_pending_exit(&bc->pending[1]);  in bch2_fs_btree_key_cache_exit()
744 	free_percpu(bc->nr_pending);  in bch2_fs_btree_key_cache_exit()
751 int bch2_fs_btree_key_cache_init(struct btree_key_cache *bc)  in bch2_fs_btree_key_cache_init()  argument
753 	struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache);  in bch2_fs_btree_key_cache_init()
756 	bc->nr_pending = alloc_percpu(size_t);  in bch2_fs_btree_key_cache_init()
757 	if (!bc->nr_pending)  in bch2_fs_btree_key_cache_init()
760 	if (rcu_pending_init(&bc->pending[0], &c->btree_trans_barrier, __bkey_cached_free) ||  in bch2_fs_btree_key_cache_init()
761 	    rcu_pending_init(&bc->pending[1], &c->btree_trans_barrier, __bkey_cached_free))  in bch2_fs_btree_key_cache_init()
764 	if (rhashtable_init(&bc->table, &bch2_btree_key_cache_params))  in bch2_fs_btree_key_cache_init()
767 	bc->table_init_done = true;  in bch2_fs_btree_key_cache_init()
772 	bc->shrink = shrink;  in bch2_fs_btree_key_cache_init()
782 void bch2_btree_key_cache_to_text(struct printbuf *out, struct btree_key_cache *bc)  in bch2_btree_key_cache_to_text()  argument
787 	prt_printf(out, "keys:\t%lu\r\n",		atomic_long_read(&bc->nr_keys));  in bch2_btree_key_cache_to_text()
788 	prt_printf(out, "dirty:\t%lu\r\n",		atomic_long_read(&bc->nr_dirty));  in bch2_btree_key_cache_to_text()
789 	prt_printf(out, "table size:\t%u\r\n",		bc->table.tbl->size);  in bch2_btree_key_cache_to_text()
792 	prt_printf(out, "requested_to_free:\t%lu\r\n",	bc->requested_to_free);  in bch2_btree_key_cache_to_text()
793 	prt_printf(out, "freed:\t%lu\r\n",		bc->freed);  in bch2_btree_key_cache_to_text()
794 	prt_printf(out, "skipped_dirty:\t%lu\r\n",	bc->skipped_dirty);  in bch2_btree_key_cache_to_text()
795 	prt_printf(out, "skipped_accessed:\t%lu\r\n",	bc->skipped_accessed);  in bch2_btree_key_cache_to_text()
796 	prt_printf(out, "skipped_lock_fail:\t%lu\r\n",	bc->skipped_lock_fail);  in bch2_btree_key_cache_to_text()
798 	prt_printf(out, "pending:\t%zu\r\n",		per_cpu_sum(bc->nr_pending));  in bch2_btree_key_cache_to_text()