1 // SPDX-License-Identifier: GPL-2.0
19  * We have two parallel sets of counters to be concerned with, and both must be
22  *  - Persistent/on disk accounting, stored in the accounting btree and updated
23  *    via btree write buffer updates that treat new accounting keys as deltas to
24  *    apply to existing values. But reading from a write buffer btree is
27  *  - In memory accounting, where accounting is stored as an array of percpu
31  *    Cheap to read, but non persistent.
35  * the output of these triggers are deltas to the various counters.
38  * counters in the disk accounting key are deltas that will be applied to the
42  * To do a disk accounting update:
43  * - initialize a disk_accounting_pos, to specify which counter is being update
44  * - initialize counter deltas, as an array of 1-3 s64s
45  * - call bch2_disk_accounting_mod()
47  * This queues up the accounting update to be done at transaction commit time.
50  * The transaction commit path is responsible for propagating updates to the in
55  * buffer; this is used by journal replay to determine which updates have been
61  * bch2_accounting_mem_mod() tells it to.
71 static inline void accounting_key_init(struct bkey_i *k, struct disk_accounting_pos *pos,  in accounting_key_init()  argument
74 	struct bkey_i_accounting *acc = bkey_accounting_init(k);  in accounting_key_init()
76 	acc->k.p = disk_accounting_pos_to_bpos(pos);  in accounting_key_init()
77 	set_bkey_val_u64s(&acc->k, sizeof(struct bch_accounting) / sizeof(u64) + nr);  in accounting_key_init()
79 	memcpy_u64s_small(acc->v.d, d, nr);  in accounting_key_init()
83 			     struct disk_accounting_pos *k,  in bch2_disk_accounting_mod()  argument
87 	switch (k->type) {  in bch2_disk_accounting_mod()
89 		bubble_sort(k->replicas.devs, k->replicas.nr_devs, u8_cmp);  in bch2_disk_accounting_mod()
95 	struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i;  in bch2_disk_accounting_mod()
97 	accounting_key_init(&k_i.k, k, d, nr);  in bch2_disk_accounting_mod()
100 		? bch2_trans_update_buffered(trans, BTREE_ID_accounting, &k_i.k)  in bch2_disk_accounting_mod()
101 		: bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(&k_i.k), true);  in bch2_disk_accounting_mod()
129 int bch2_accounting_validate(struct bch_fs *c, struct bkey_s_c k,  in bch2_accounting_validate()  argument
133 	bpos_to_disk_accounting_pos(&acc_k, k.k->p);  in bch2_accounting_validate()
137 	bkey_fsck_err_on(bversion_zero(k.k->bversion),  in bch2_accounting_validate()
190 void bch2_accounting_key_to_text(struct printbuf *out, struct disk_accounting_pos *k)  in bch2_accounting_key_to_text()  argument
192 	if (k->type >= BCH_DISK_ACCOUNTING_TYPE_NR) {  in bch2_accounting_key_to_text()
193 		prt_printf(out, "unknown type %u", k->type);  in bch2_accounting_key_to_text()
197 	prt_str(out, disk_accounting_type_strs[k->type]);  in bch2_accounting_key_to_text()
200 	switch (k->type) {  in bch2_accounting_key_to_text()
204 		prt_printf(out, "replicas=%u", k->persistent_reserved.nr_replicas);  in bch2_accounting_key_to_text()
207 		bch2_replicas_entry_to_text(out, &k->replicas);  in bch2_accounting_key_to_text()
210 		prt_printf(out, "dev=%u data_type=", k->dev_data_type.dev);  in bch2_accounting_key_to_text()
211 		bch2_prt_data_type(out, k->dev_data_type.data_type);  in bch2_accounting_key_to_text()
214 		bch2_prt_compression_type(out, k->compression.type);  in bch2_accounting_key_to_text()
217 		prt_printf(out, "id=%u", k->snapshot.id);  in bch2_accounting_key_to_text()
220 		prt_printf(out, "btree=%s", bch2_btree_id_str(k->btree.id));  in bch2_accounting_key_to_text()
225 void bch2_accounting_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k)  in bch2_accounting_to_text()  argument
227 	struct bkey_s_c_accounting acc = bkey_s_c_to_accounting(k);  in bch2_accounting_to_text()
229 	bpos_to_disk_accounting_pos(&acc_k, k.k->p);  in bch2_accounting_to_text()
233 	for (unsigned i = 0; i < bch2_accounting_counters(k.k); i++)  in bch2_accounting_to_text()
234 		prt_printf(out, " %lli", acc.v->d[i]);  in bch2_accounting_to_text()
237 void bch2_accounting_swab(struct bkey_s k)  in bch2_accounting_swab()  argument
239 	for (u64 *p = (u64 *) k.v;  in bch2_accounting_swab()
240 	     p < (u64 *) bkey_val_end(k);  in bch2_accounting_swab()
281 	for (struct jset_entry *i = trans->journal_entries;  in bch2_accounting_update_sb()
282 	     i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);  in bch2_accounting_update_sb()
284 		if (jset_entry_is_key(i) && i->start->k.type == KEY_TYPE_accounting) {  in bch2_accounting_update_sb()
285 			int ret = bch2_accounting_update_sb_one(trans->c, i->start->k.p);  in bch2_accounting_update_sb()
295 	struct bch_accounting_mem *acc = &c->accounting;  in __bch2_accounting_mem_insert()
298 	if (eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),  in __bch2_accounting_mem_insert()
299 			    accounting_pos_cmp, &a.k->p) < acc->k.nr)  in __bch2_accounting_mem_insert()
303 		.pos		= a.k->p,  in __bch2_accounting_mem_insert()
304 		.bversion	= a.k->bversion,  in __bch2_accounting_mem_insert()
305 		.nr_counters	= bch2_accounting_counters(a.k),  in __bch2_accounting_mem_insert()
313 	if (acc->gc_running) {  in __bch2_accounting_mem_insert()
320 	if (darray_push(&acc->k, n))  in __bch2_accounting_mem_insert()
323 	eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),  in __bch2_accounting_mem_insert()
329 	return -BCH_ERR_ENOMEM_disk_accounting;  in __bch2_accounting_mem_insert()
338 	    accounting_to_replicas(&r.e, a.k->p) &&  in bch2_accounting_mem_insert()
340 		return -BCH_ERR_btree_insert_need_mark_replicas;  in bch2_accounting_mem_insert()
342 	percpu_up_read(&c->mark_lock);  in bch2_accounting_mem_insert()
343 	percpu_down_write(&c->mark_lock);  in bch2_accounting_mem_insert()
345 	percpu_up_write(&c->mark_lock);  in bch2_accounting_mem_insert()
346 	percpu_down_read(&c->mark_lock);  in bch2_accounting_mem_insert()
352 	for (unsigned i = 0; i < e->nr_counters; i++)  in accounting_mem_entry_is_zero()
353 		if (percpu_u64_get(e->v[0] + i) ||  in accounting_mem_entry_is_zero()
354 		    (e->v[1] &&  in accounting_mem_entry_is_zero()
355 		     percpu_u64_get(e->v[1] + i)))  in accounting_mem_entry_is_zero()
362 	struct bch_accounting_mem *acc = &c->accounting;  in bch2_accounting_mem_gc()
364 	percpu_down_write(&c->mark_lock);  in bch2_accounting_mem_gc()
365 	struct accounting_mem_entry *dst = acc->k.data;  in bch2_accounting_mem_gc()
367 	darray_for_each(acc->k, src) {  in bch2_accounting_mem_gc()
369 			free_percpu(src->v[0]);  in bch2_accounting_mem_gc()
370 			free_percpu(src->v[1]);  in bch2_accounting_mem_gc()
376 	acc->k.nr = dst - acc->k.data;  in bch2_accounting_mem_gc()
377 	eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),  in bch2_accounting_mem_gc()
379 	percpu_up_write(&c->mark_lock);  in bch2_accounting_mem_gc()
387  * with something more general, it exists to support the ioctl used by the
392 	struct bch_accounting_mem *acc = &c->accounting;  in bch2_fs_replicas_usage_read()
397 	percpu_down_read(&c->mark_lock);  in bch2_fs_replicas_usage_read()
398 	darray_for_each(acc->k, i) {  in bch2_fs_replicas_usage_read()
404 		if (!accounting_to_replicas(&u.r.r, i->pos))  in bch2_fs_replicas_usage_read()
408 		bch2_accounting_mem_read_counters(acc, i - acc->k.data, &sectors, 1, false);  in bch2_fs_replicas_usage_read()
416 		usage->nr += replicas_usage_bytes(&u.r);  in bch2_fs_replicas_usage_read()
418 	percpu_up_read(&c->mark_lock);  in bch2_fs_replicas_usage_read()
428 	struct bch_accounting_mem *acc = &c->accounting;  in bch2_fs_accounting_read()
433 	percpu_down_read(&c->mark_lock);  in bch2_fs_accounting_read()
434 	darray_for_each(acc->k, i) {  in bch2_fs_accounting_read()
436 		bpos_to_disk_accounting_pos(&a_p, i->pos);  in bch2_fs_accounting_read()
442 				       sizeof(u64) * i->nr_counters);  in bch2_fs_accounting_read()
448 		set_bkey_val_u64s(&a_out->k, i->nr_counters);  in bch2_fs_accounting_read()
449 		a_out->k.p = i->pos;  in bch2_fs_accounting_read()
450 		bch2_accounting_mem_read_counters(acc, i - acc->k.data,  in bch2_fs_accounting_read()
451 						  a_out->v.d, i->nr_counters, false);  in bch2_fs_accounting_read()
454 			out_buf->nr += bkey_bytes(&a_out->k);  in bch2_fs_accounting_read()
457 	percpu_up_read(&c->mark_lock);  in bch2_fs_accounting_read()
466 	struct bch_accounting_mem *acc = &c->accounting;  in bch2_fs_accounting_to_text()
468 	percpu_down_read(&c->mark_lock);  in bch2_fs_accounting_to_text()
469 	out->atomic++;  in bch2_fs_accounting_to_text()
471 	eytzinger0_for_each(i, acc->k.nr) {  in bch2_fs_accounting_to_text()
473 		bpos_to_disk_accounting_pos(&acc_k, acc->k.data[i].pos);  in bch2_fs_accounting_to_text()
481 		for (unsigned j = 0; j < acc->k.data[i].nr_counters; j++)  in bch2_fs_accounting_to_text()  local
482 			prt_printf(out, " %llu", v[j]);  in bch2_fs_accounting_to_text()
486 	--out->atomic;  in bch2_fs_accounting_to_text()
487 	percpu_up_read(&c->mark_lock);  in bch2_fs_accounting_to_text()
492 	darray_for_each(acc->k, e) {  in bch2_accounting_free_counters()
493 		free_percpu(e->v[gc]);  in bch2_accounting_free_counters()
494 		e->v[gc] = NULL;  in bch2_accounting_free_counters()
500 	struct bch_accounting_mem *acc = &c->accounting;  in bch2_gc_accounting_start()
503 	percpu_down_write(&c->mark_lock);  in bch2_gc_accounting_start()
504 	darray_for_each(acc->k, e) {  in bch2_gc_accounting_start()
505 		e->v[1] = __alloc_percpu_gfp(e->nr_counters * sizeof(u64),  in bch2_gc_accounting_start()
507 		if (!e->v[1]) {  in bch2_gc_accounting_start()
509 			ret = -BCH_ERR_ENOMEM_disk_accounting;  in bch2_gc_accounting_start()
514 	acc->gc_running = !ret;  in bch2_gc_accounting_start()
515 	percpu_up_write(&c->mark_lock);  in bch2_gc_accounting_start()
522 	struct bch_accounting_mem *acc = &c->accounting;  in bch2_gc_accounting_done()
528 	percpu_down_write(&c->mark_lock);  in bch2_gc_accounting_done()
530 		unsigned idx = eytzinger0_find_ge(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),  in bch2_gc_accounting_done()
533 		if (idx >= acc->k.nr)  in bch2_gc_accounting_done()
536 		struct accounting_mem_entry *e = acc->k.data + idx;  in bch2_gc_accounting_done()
537 		pos = bpos_successor(e->pos);  in bch2_gc_accounting_done()
540 		bpos_to_disk_accounting_pos(&acc_k, e->pos);  in bch2_gc_accounting_done()
548 		unsigned nr = e->nr_counters;  in bch2_gc_accounting_done()
558 			for (unsigned j = 0; j < nr; j++)  in bch2_gc_accounting_done()  local
559 				prt_printf(&buf, " %llu", dst_v[j]);  in bch2_gc_accounting_done()
562 			for (unsigned j = 0; j < nr; j++)  in bch2_gc_accounting_done()  local
563 				prt_printf(&buf, " %llu", src_v[j]);  in bch2_gc_accounting_done()
565 			for (unsigned j = 0; j < nr; j++)  in bch2_gc_accounting_done()  local
566 				src_v[j] -= dst_v[j];  in bch2_gc_accounting_done()
569 				percpu_up_write(&c->mark_lock);  in bch2_gc_accounting_done()
572 				percpu_down_write(&c->mark_lock);  in bch2_gc_accounting_done()
576 				if (!test_bit(BCH_FS_may_go_rw, &c->flags)) {  in bch2_gc_accounting_done()
577 					memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta));  in bch2_gc_accounting_done()
578 					struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i;  in bch2_gc_accounting_done()
580 					accounting_key_init(&k_i.k, &acc_k, src_v, nr);  in bch2_gc_accounting_done()
582 								bkey_i_to_s_c_accounting(&k_i.k),  in bch2_gc_accounting_done()
586 					struct bch_fs_usage_base *dst = this_cpu_ptr(c->usage);  in bch2_gc_accounting_done()
587 					struct bch_fs_usage_base *src = &trans->fs_usage_delta;  in bch2_gc_accounting_done()
596 	percpu_up_write(&c->mark_lock);  in bch2_gc_accounting_done()
603 static int accounting_read_key(struct btree_trans *trans, struct bkey_s_c k)  in accounting_read_key()  argument
605 	struct bch_fs *c = trans->c;  in accounting_read_key()
607 	if (k.k->type != KEY_TYPE_accounting)  in accounting_read_key()
610 	percpu_down_read(&c->mark_lock);  in accounting_read_key()
611 	int ret = bch2_accounting_mem_mod_locked(trans, bkey_s_c_to_accounting(k),  in accounting_read_key()
613 	percpu_up_read(&c->mark_lock);  in accounting_read_key()
621 	struct bch_fs *c = trans->c;  in bch2_disk_accounting_validate_late()
623 	int ret = 0, invalid_dev = -1;  in bch2_disk_accounting_validate_late()
653 			percpu_up_write(&c->mark_lock);  in bch2_disk_accounting_validate_late()
657 			percpu_down_write(&c->mark_lock);  in bch2_disk_accounting_validate_late()
675 		     "accounting entry points to invalid device %i\n  %s",  in bch2_disk_accounting_validate_late()
681 			v[i] = -v[i];  in bch2_disk_accounting_validate_late()
685 			-BCH_ERR_remove_disk_accounting_entry;  in bch2_disk_accounting_validate_late()
687 		ret = -BCH_ERR_remove_disk_accounting_entry;  in bch2_disk_accounting_validate_late()
698 	struct bch_accounting_mem *acc = &c->accounting;  in bch2_accounting_read()
704 				BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, ({  in bch2_accounting_read()
706 			struct bkey_s_c k = bch2_btree_path_peek_slot_exact(btree_iter_path(trans, &iter), &u);  in bch2_accounting_read()
707 			accounting_read_key(trans, k);  in bch2_accounting_read()
712 	struct journal_keys *keys = &c->journal_keys;  in bch2_accounting_read()
713 	struct journal_key *dst = keys->data;  in bch2_accounting_read()
714 	move_gap(keys, keys->nr);  in bch2_accounting_read()
717 		if (i->k->k.type == KEY_TYPE_accounting) {  in bch2_accounting_read()
718 			struct bkey_s_c k = bkey_i_to_s_c(i->k);  in bch2_accounting_read()  local
719 			unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr,  in bch2_accounting_read()
720 						sizeof(acc->k.data[0]),  in bch2_accounting_read()
721 						accounting_pos_cmp, &k.k->p);  in bch2_accounting_read()
723 			bool applied = idx < acc->k.nr &&  in bch2_accounting_read()
724 				bversion_cmp(acc->k.data[idx].bversion, k.k->bversion) >= 0;  in bch2_accounting_read()
730 			    i[1].k->k.type == KEY_TYPE_accounting &&  in bch2_accounting_read()
732 				WARN_ON(bversion_cmp(i[0].k->k.bversion, i[1].k->k.bversion) >= 0);  in bch2_accounting_read()
736 				bch2_accounting_accumulate(bkey_i_to_accounting(i[1].k),  in bch2_accounting_read()
737 							   bkey_s_c_to_accounting(k));  in bch2_accounting_read()
741 			ret = accounting_read_key(trans, k);  in bch2_accounting_read()
748 	keys->gap = keys->nr = dst - keys->data;  in bch2_accounting_read()
750 	percpu_down_write(&c->mark_lock);  in bch2_accounting_read()
752 	while (i < acc->k.nr) {  in bch2_accounting_read()
753 		unsigned idx = inorder_to_eytzinger0(i, acc->k.nr);  in bch2_accounting_read()
756 		bpos_to_disk_accounting_pos(&acc_k, acc->k.data[idx].pos);  in bch2_accounting_read()
763 		 * nonexistent - it might point to an invalid device.  in bch2_accounting_read()
765 		 * Remove it, so that if it's re-added it gets re-marked in the  in bch2_accounting_read()
768 		ret = bch2_is_zero(v, sizeof(v[0]) * acc->k.data[idx].nr_counters)  in bch2_accounting_read()
769 			? -BCH_ERR_remove_disk_accounting_entry  in bch2_accounting_read()
771 							v, acc->k.data[idx].nr_counters);  in bch2_accounting_read()
773 		if (ret == -BCH_ERR_remove_disk_accounting_entry) {  in bch2_accounting_read()
774 			free_percpu(acc->k.data[idx].v[0]);  in bch2_accounting_read()
775 			free_percpu(acc->k.data[idx].v[1]);  in bch2_accounting_read()
776 			darray_remove_item(&acc->k, &acc->k.data[idx]);  in bch2_accounting_read()
777 			eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),  in bch2_accounting_read()
789 	struct bch_fs_usage_base *usage = this_cpu_ptr(c->usage);  in bch2_accounting_read()
791 	for (unsigned i = 0; i < acc->k.nr; i++) {  in bch2_accounting_read()
792 		struct disk_accounting_pos k;  in bch2_accounting_read()  local
793 		bpos_to_disk_accounting_pos(&k, acc->k.data[i].pos);  in bch2_accounting_read()
798 		switch (k.type) {  in bch2_accounting_read()
800 			usage->reserved += v[0] * k.persistent_reserved.nr_replicas;  in bch2_accounting_read()
803 			fs_usage_data_type_to_base(usage, k.replicas.data_type, v[0]);  in bch2_accounting_read()
807 			struct bch_dev *ca = bch2_dev_rcu(c, k.dev_data_type.dev);  in bch2_accounting_read()
809 				struct bch_dev_usage_type __percpu *d = &ca->usage->d[k.dev_data_type.data_type];  in bch2_accounting_read()
810 				percpu_u64_set(&d->buckets,	v[0]);  in bch2_accounting_read()
811 				percpu_u64_set(&d->sectors,	v[1]);  in bch2_accounting_read()
812 				percpu_u64_set(&d->fragmented,	v[2]);  in bch2_accounting_read()
814 				if (k.dev_data_type.data_type == BCH_DATA_sb ||  in bch2_accounting_read()
815 				    k.dev_data_type.data_type == BCH_DATA_journal)  in bch2_accounting_read()
816 					usage->hidden += v[0] * ca->mi.bucket_size;  in bch2_accounting_read()
824 	percpu_up_write(&c->mark_lock);  in bch2_accounting_read()
837 				BTREE_ITER_all_snapshots, k, NULL, NULL, 0, ({  in bch2_dev_usage_remove()
839 			bpos_to_disk_accounting_pos(&acc, k.k->p);  in bch2_dev_usage_remove()
843 				? bch2_btree_bit_mod_buffered(trans, BTREE_ID_accounting, k.k->p, 0)  in bch2_dev_usage_remove()
851 	struct bch_fs *c = ca->fs;  in bch2_dev_usage_init()
854 		.dev_data_type.dev = ca->dev_idx,  in bch2_dev_usage_init()
857 	u64 v[3] = { ca->mi.nbuckets - ca->mi.first_bucket, 0, 0 };  in bch2_dev_usage_init()
875 				   BTREE_ITER_all_snapshots, k, ({  in bch2_verify_accounting_clean()
877 			struct bkey_s_c_accounting a = bkey_s_c_to_accounting(k);  in bch2_verify_accounting_clean()
878 			unsigned nr = bch2_accounting_counters(k.k);  in bch2_verify_accounting_clean()
881 			bpos_to_disk_accounting_pos(&acc_k, k.k->p);  in bch2_verify_accounting_clean()
889 			bch2_accounting_mem_read(c, k.k->p, v, nr);  in bch2_verify_accounting_clean()
891 			if (memcmp(a.v->d, v, nr * sizeof(u64))) {  in bch2_verify_accounting_clean()
894 				bch2_bkey_val_to_text(&buf, c, k);  in bch2_verify_accounting_clean()
896 				for (unsigned j = 0; j < nr; j++)  in bch2_verify_accounting_clean()
897 					prt_printf(&buf, " %llu", v[j]);  in bch2_verify_accounting_clean()
906 				base.reserved += acc_k.persistent_reserved.nr_replicas * a.v->d[0];  in bch2_verify_accounting_clean()
909 				fs_usage_data_type_to_base(&base, acc_k.replicas.data_type, a.v->d[0]);  in bch2_verify_accounting_clean()
919 				v[0] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].buckets);  in bch2_verify_accounting_clean()
920 				v[1] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].sectors);  in bch2_verify_accounting_clean()
921 				v[2] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].fragmented);  in bch2_verify_accounting_clean()
924 				if (memcmp(a.v->d, v, 3 * sizeof(u64))) {  in bch2_verify_accounting_clean()
927 					bch2_bkey_val_to_text(&buf, c, k);  in bch2_verify_accounting_clean()
929 					for (unsigned j = 0; j < nr; j++)  in bch2_verify_accounting_clean()
930 						prt_printf(&buf, " %llu", v[j]);  in bch2_verify_accounting_clean()
942 	acc_u64s_percpu(&base_inmem.hidden, &c->usage->hidden, sizeof(base_inmem) / sizeof(u64));  in bch2_verify_accounting_clean()
962 	lockdep_assert_held(&c->mark_lock);  in bch2_accounting_gc_free()
964 	struct bch_accounting_mem *acc = &c->accounting;  in bch2_accounting_gc_free()
967 	acc->gc_running = false;  in bch2_accounting_gc_free()
972 	struct bch_accounting_mem *acc = &c->accounting;  in bch2_fs_accounting_exit()
975 	darray_exit(&acc->k);  in bch2_fs_accounting_exit()