67 static inline atomic_t *get_zone_count_ptr(struct recovery_journal *journal,  in get_zone_count_ptr()  argument
72 		? &journal->lock_counter.logical_zone_counts[lock_number]  in get_zone_count_ptr()
73 		: &journal->lock_counter.physical_zone_counts[lock_number]);  in get_zone_count_ptr()
85 static inline u16 *get_counter(struct recovery_journal *journal,  in get_counter()  argument
89 	struct lock_counter *counter = &journal->lock_counter;  in get_counter()
101 static atomic_t *get_decrement_counter(struct recovery_journal *journal,  in get_decrement_counter()  argument
104 	return &journal->lock_counter.journal_decrement_counts[lock_number];  in get_decrement_counter()
114 static bool is_journal_zone_locked(struct recovery_journal *journal,  in is_journal_zone_locked()  argument
117 	u16 journal_value = *get_counter(journal, lock_number, VDO_ZONE_TYPE_JOURNAL, 0);  in is_journal_zone_locked()
118 	u32 decrements = atomic_read(get_decrement_counter(journal, lock_number));  in is_journal_zone_locked()
138 void vdo_release_recovery_journal_block_reference(struct recovery_journal *journal,  in vdo_release_recovery_journal_block_reference()  argument
150 	lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number);  in vdo_release_recovery_journal_block_reference()
151 	current_value = get_counter(journal, lock_number, zone_type, zone_id);  in vdo_release_recovery_journal_block_reference()
158 		if (is_journal_zone_locked(journal, lock_number))  in vdo_release_recovery_journal_block_reference()
166 		zone_count = get_zone_count_ptr(journal, lock_number, zone_type);  in vdo_release_recovery_journal_block_reference()
177 	prior_state = atomic_cmpxchg(&journal->lock_counter.state,  in vdo_release_recovery_journal_block_reference()
186 	vdo_launch_completion(&journal->lock_counter.completion);  in vdo_release_recovery_journal_block_reference()
200 static struct recovery_journal_block * __must_check pop_free_list(struct recovery_journal *journal)  in pop_free_list()  argument
204 	if (list_empty(&journal->free_tail_blocks))  in pop_free_list()
207 	block = list_last_entry(&journal->free_tail_blocks,  in pop_free_list()
246 	return ((block == NULL) || (block->journal->entries_per_block == block->entry_count));  in is_block_full()
254 static void assert_on_journal_thread(struct recovery_journal *journal,  in assert_on_journal_thread()  argument
257 	VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == journal->thread_id),  in assert_on_journal_thread()
278 static inline bool has_block_waiters(struct recovery_journal *journal)  in has_block_waiters()  argument
280 	struct recovery_journal_block *block = get_journal_block(&journal->active_tail_blocks);  in has_block_waiters()
291 static void recycle_journal_blocks(struct recovery_journal *journal);
293 static void notify_commit_waiters(struct recovery_journal *journal);
319 static inline bool is_read_only(struct recovery_journal *journal)  in is_read_only()  argument
321 	return vdo_is_read_only(journal->flush_vio->completion.vdo);  in is_read_only()
328 static void check_for_drain_complete(struct recovery_journal *journal)  in check_for_drain_complete()  argument
332 	if (is_read_only(journal)) {  in check_for_drain_complete()
339 		notify_commit_waiters(journal);  in check_for_drain_complete()
340 		recycle_journal_blocks(journal);  in check_for_drain_complete()
343 		vdo_waitq_notify_all_waiters(&journal->entry_waiters,  in check_for_drain_complete()
347 	if (!vdo_is_state_draining(&journal->state) ||  in check_for_drain_complete()
348 	    journal->reaping ||  in check_for_drain_complete()
349 	    has_block_waiters(journal) ||  in check_for_drain_complete()
350 	    vdo_waitq_has_waiters(&journal->entry_waiters) ||  in check_for_drain_complete()
351 	    !suspend_lock_counter(&journal->lock_counter))  in check_for_drain_complete()
354 	if (vdo_is_state_saving(&journal->state)) {  in check_for_drain_complete()
355 		if (journal->active_block != NULL) {  in check_for_drain_complete()
357 					     !is_block_dirty(journal->active_block)),  in check_for_drain_complete()
359 			recycle_journal_block(journal->active_block);  in check_for_drain_complete()
362 		VDO_ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks),  in check_for_drain_complete()
366 	vdo_finish_draining_with_result(&journal->state, result);  in check_for_drain_complete()
392 static void enter_journal_read_only_mode(struct recovery_journal *journal,  in enter_journal_read_only_mode()  argument
395 	vdo_enter_read_only_mode(journal->flush_vio->completion.vdo, error_code);  in enter_journal_read_only_mode()
396 	check_for_drain_complete(journal);  in enter_journal_read_only_mode()
408 sequence_number_t vdo_get_recovery_journal_current_sequence_number(struct recovery_journal *journal)  in vdo_get_recovery_journal_current_sequence_number()  argument
410 	return journal->tail;  in vdo_get_recovery_journal_current_sequence_number()
421 static inline sequence_number_t get_recovery_journal_head(const struct recovery_journal *journal)  in get_recovery_journal_head()  argument
423 	return min(journal->block_map_head, journal->slab_journal_head);  in get_recovery_journal_head()
442 static void check_slab_journal_commit_threshold(struct recovery_journal *journal)  in check_slab_journal_commit_threshold()  argument
444 	block_count_t current_length = journal->tail - journal->slab_journal_head;  in check_slab_journal_commit_threshold()
446 	if (current_length > journal->slab_journal_commit_threshold) {  in check_slab_journal_commit_threshold()
447 		journal->events.slab_journal_commits_requested++;  in check_slab_journal_commit_threshold()
448 		vdo_commit_oldest_slab_journal_tail_blocks(journal->depot,  in check_slab_journal_commit_threshold()
449 							   journal->slab_journal_head);  in check_slab_journal_commit_threshold()
453 static void reap_recovery_journal(struct recovery_journal *journal);
454 static void assign_entries(struct recovery_journal *journal);
460 static void finish_reaping(struct recovery_journal *journal)  in finish_reaping()  argument
463 	sequence_number_t old_head = get_recovery_journal_head(journal);  in finish_reaping()
465 	journal->block_map_head = journal->block_map_reap_head;  in finish_reaping()
466 	journal->slab_journal_head = journal->slab_journal_reap_head;  in finish_reaping()
467 	blocks_reaped = get_recovery_journal_head(journal) - old_head;  in finish_reaping()
468 	journal->available_space += blocks_reaped * journal->entries_per_block;  in finish_reaping()
469 	journal->reaping = false;  in finish_reaping()
470 	check_slab_journal_commit_threshold(journal);  in finish_reaping()
471 	assign_entries(journal);  in finish_reaping()
472 	check_for_drain_complete(journal);  in finish_reaping()
483 	struct recovery_journal *journal = completion->parent;  in complete_reaping()  local
485 	finish_reaping(journal);  in complete_reaping()
488 	reap_recovery_journal(journal);  in complete_reaping()
497 	struct recovery_journal *journal = completion->parent;  in handle_flush_error()  local
500 	journal->reaping = false;  in handle_flush_error()
501 	enter_journal_read_only_mode(journal, completion->result);  in handle_flush_error()
507 	struct recovery_journal *journal = vio->completion.parent;  in flush_endio()  local
509 	continue_vio_after_io(vio, complete_reaping, journal->thread_id);  in flush_endio()
517 static void initialize_journal_state(struct recovery_journal *journal)  in initialize_journal_state()  argument
519 	journal->append_point.sequence_number = journal->tail;  in initialize_journal_state()
520 	journal->last_write_acknowledged = journal->tail;  in initialize_journal_state()
521 	journal->block_map_head = journal->tail;  in initialize_journal_state()
522 	journal->slab_journal_head = journal->tail;  in initialize_journal_state()
523 	journal->block_map_reap_head = journal->tail;  in initialize_journal_state()
524 	journal->slab_journal_reap_head = journal->tail;  in initialize_journal_state()
525 	journal->block_map_head_block_number =  in initialize_journal_state()
526 		vdo_get_recovery_journal_block_number(journal, journal->block_map_head);  in initialize_journal_state()
527 	journal->slab_journal_head_block_number =  in initialize_journal_state()
528 		vdo_get_recovery_journal_block_number(journal,  in initialize_journal_state()
529 						      journal->slab_journal_head);  in initialize_journal_state()
530 	journal->available_space =  in initialize_journal_state()
531 		(journal->entries_per_block * vdo_get_recovery_journal_length(journal->size));  in initialize_journal_state()
558 	struct recovery_journal *journal = (struct recovery_journal *) completion->parent;  in reap_recovery_journal_callback()  local
564 	atomic_set(&journal->lock_counter.state, LOCK_COUNTER_STATE_NOT_NOTIFYING);  in reap_recovery_journal_callback()
566 	if (vdo_is_state_quiescing(&journal->state)) {  in reap_recovery_journal_callback()
571 		check_for_drain_complete(journal);  in reap_recovery_journal_callback()
575 	reap_recovery_journal(journal);  in reap_recovery_journal_callback()
576 	check_slab_journal_commit_threshold(journal);  in reap_recovery_journal_callback()
587 static int __must_check initialize_lock_counter(struct recovery_journal *journal,  in initialize_lock_counter()  argument
592 	struct lock_counter *counter = &journal->lock_counter;  in initialize_lock_counter()
594 	result = vdo_allocate(journal->size, u16, __func__, &counter->journal_counters);  in initialize_lock_counter()
598 	result = vdo_allocate(journal->size, atomic_t, __func__,  in initialize_lock_counter()
603 	result = vdo_allocate(journal->size * config->logical_zone_count, u16, __func__,  in initialize_lock_counter()
608 	result = vdo_allocate(journal->size, atomic_t, __func__,  in initialize_lock_counter()
613 	result = vdo_allocate(journal->size * config->physical_zone_count, u16, __func__,  in initialize_lock_counter()
618 	result = vdo_allocate(journal->size, atomic_t, __func__,  in initialize_lock_counter()
627 			       journal);  in initialize_lock_counter()
630 	counter->locks = journal->size;  in initialize_lock_counter()
639 static void set_journal_tail(struct recovery_journal *journal, sequence_number_t tail)  in set_journal_tail()  argument
643 		enter_journal_read_only_mode(journal, VDO_JOURNAL_OVERFLOW);  in set_journal_tail()
645 	journal->tail = tail;  in set_journal_tail()
656 static int initialize_recovery_block(struct vdo *vdo, struct recovery_journal *journal,  in initialize_recovery_block()  argument
684 	list_add_tail(&block->list_node, &journal->free_tail_blocks);  in initialize_recovery_block()
685 	block->journal = journal;  in initialize_recovery_block()
709 	struct recovery_journal *journal;  in vdo_decode_recovery_journal()  local
715 				       &journal);  in vdo_decode_recovery_journal()
719 	INIT_LIST_HEAD(&journal->free_tail_blocks);  in vdo_decode_recovery_journal()
720 	INIT_LIST_HEAD(&journal->active_tail_blocks);  in vdo_decode_recovery_journal()
721 	vdo_waitq_init(&journal->pending_writes);  in vdo_decode_recovery_journal()
723 	journal->thread_id = vdo->thread_config.journal_thread;  in vdo_decode_recovery_journal()
724 	journal->origin = partition->offset;  in vdo_decode_recovery_journal()
725 	journal->nonce = nonce;  in vdo_decode_recovery_journal()
726 	journal->recovery_count = compute_recovery_count_byte(recovery_count);  in vdo_decode_recovery_journal()
727 	journal->size = journal_size;  in vdo_decode_recovery_journal()
728 	journal->slab_journal_commit_threshold = (journal_size * 2) / 3;  in vdo_decode_recovery_journal()
729 	journal->logical_blocks_used = state.logical_blocks_used;  in vdo_decode_recovery_journal()
730 	journal->block_map_data_blocks = state.block_map_data_blocks;  in vdo_decode_recovery_journal()
731 	journal->entries_per_block = RECOVERY_JOURNAL_ENTRIES_PER_BLOCK;  in vdo_decode_recovery_journal()
732 	set_journal_tail(journal, state.journal_start);  in vdo_decode_recovery_journal()
733 	initialize_journal_state(journal);  in vdo_decode_recovery_journal()
735 	vdo_set_admin_state_code(&journal->state, VDO_ADMIN_STATE_SUSPENDED);  in vdo_decode_recovery_journal()
738 		struct recovery_journal_block *block = &journal->blocks[i];  in vdo_decode_recovery_journal()
740 		result = initialize_recovery_block(vdo, journal, block);  in vdo_decode_recovery_journal()
742 			vdo_free_recovery_journal(journal);  in vdo_decode_recovery_journal()
747 	result = initialize_lock_counter(journal, vdo);  in vdo_decode_recovery_journal()
749 		vdo_free_recovery_journal(journal);  in vdo_decode_recovery_journal()
754 				     journal, NULL, &journal->flush_vio);  in vdo_decode_recovery_journal()
756 		vdo_free_recovery_journal(journal);  in vdo_decode_recovery_journal()
760 	result = vdo_register_read_only_listener(vdo, journal,  in vdo_decode_recovery_journal()
762 						 journal->thread_id);  in vdo_decode_recovery_journal()
764 		vdo_free_recovery_journal(journal);  in vdo_decode_recovery_journal()
768 	result = vdo_make_default_thread(vdo, journal->thread_id);  in vdo_decode_recovery_journal()
770 		vdo_free_recovery_journal(journal);  in vdo_decode_recovery_journal()
774 	journal->flush_vio->completion.callback_thread_id = journal->thread_id;  in vdo_decode_recovery_journal()
775 	*journal_ptr = journal;  in vdo_decode_recovery_journal()
783 void vdo_free_recovery_journal(struct recovery_journal *journal)  in vdo_free_recovery_journal()  argument
787 	if (journal == NULL)  in vdo_free_recovery_journal()
790 	vdo_free(vdo_forget(journal->lock_counter.logical_zone_counts));  in vdo_free_recovery_journal()
791 	vdo_free(vdo_forget(journal->lock_counter.physical_zone_counts));  in vdo_free_recovery_journal()
792 	vdo_free(vdo_forget(journal->lock_counter.journal_counters));  in vdo_free_recovery_journal()
793 	vdo_free(vdo_forget(journal->lock_counter.journal_decrement_counts));  in vdo_free_recovery_journal()
794 	vdo_free(vdo_forget(journal->lock_counter.logical_counters));  in vdo_free_recovery_journal()
795 	vdo_free(vdo_forget(journal->lock_counter.physical_counters));  in vdo_free_recovery_journal()
796 	free_vio(vdo_forget(journal->flush_vio));  in vdo_free_recovery_journal()
802 	if (!vdo_is_state_quiescent(&journal->state)) {  in vdo_free_recovery_journal()
803 		VDO_ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks),  in vdo_free_recovery_journal()
805 	} else if (!vdo_is_state_saved(&journal->state) &&  in vdo_free_recovery_journal()
806 		   !list_empty(&journal->active_tail_blocks)) {  in vdo_free_recovery_journal()
811 		struct recovery_journal_block *block = &journal->blocks[i];  in vdo_free_recovery_journal()
817 	vdo_free(journal);  in vdo_free_recovery_journal()
828 void vdo_initialize_recovery_journal_post_repair(struct recovery_journal *journal,  in vdo_initialize_recovery_journal_post_repair()  argument
834 	set_journal_tail(journal, tail + 1);  in vdo_initialize_recovery_journal_post_repair()
835 	journal->recovery_count = compute_recovery_count_byte(recovery_count);  in vdo_initialize_recovery_journal_post_repair()
836 	initialize_journal_state(journal);  in vdo_initialize_recovery_journal_post_repair()
837 	journal->logical_blocks_used = logical_blocks_used;  in vdo_initialize_recovery_journal_post_repair()
838 	journal->block_map_data_blocks = block_map_data_blocks;  in vdo_initialize_recovery_journal_post_repair()
848 block_count_t vdo_get_journal_block_map_data_blocks_used(struct recovery_journal *journal)  in vdo_get_journal_block_map_data_blocks_used()  argument
850 	return journal->block_map_data_blocks;  in vdo_get_journal_block_map_data_blocks_used()
859 thread_id_t vdo_get_recovery_journal_thread_id(struct recovery_journal *journal)  in vdo_get_recovery_journal_thread_id()  argument
861 	return journal->thread_id;  in vdo_get_recovery_journal_thread_id()
870 void vdo_open_recovery_journal(struct recovery_journal *journal,  in vdo_open_recovery_journal()  argument
873 	journal->depot = depot;  in vdo_open_recovery_journal()
874 	journal->block_map = block_map;  in vdo_open_recovery_journal()
875 	WRITE_ONCE(journal->state.current_state, VDO_ADMIN_STATE_NORMAL_OPERATION);  in vdo_open_recovery_journal()
886 vdo_record_recovery_journal(const struct recovery_journal *journal)  in vdo_record_recovery_journal()  argument
889 		.logical_blocks_used = journal->logical_blocks_used,  in vdo_record_recovery_journal()
890 		.block_map_data_blocks = journal->block_map_data_blocks,  in vdo_record_recovery_journal()
893 	if (vdo_is_state_saved(&journal->state)) {  in vdo_record_recovery_journal()
898 		state.journal_start = journal->tail;  in vdo_record_recovery_journal()
904 		state.journal_start = get_recovery_journal_head(journal);  in vdo_record_recovery_journal()
931 	block->sector->recovery_count = block->journal->recovery_count;  in set_active_sector()
941 static bool advance_tail(struct recovery_journal *journal)  in advance_tail()  argument
947 	block = journal->active_block = pop_free_list(journal);  in advance_tail()
951 	list_move_tail(&block->list_node, &journal->active_tail_blocks);  in advance_tail()
955 		.block_map_data_blocks = journal->block_map_data_blocks,  in advance_tail()
956 		.logical_blocks_used = journal->logical_blocks_used,  in advance_tail()
957 		.nonce = journal->nonce,  in advance_tail()
958 		.recovery_count = journal->recovery_count,  in advance_tail()
959 		.sequence_number = journal->tail,  in advance_tail()
960 		.check_byte = vdo_compute_recovery_journal_check_byte(journal,  in advance_tail()
961 								      journal->tail),  in advance_tail()
966 	block->sequence_number = journal->tail;  in advance_tail()
969 	block->block_number = vdo_get_recovery_journal_block_number(journal,  in advance_tail()
970 								    journal->tail);  in advance_tail()
974 	set_journal_tail(journal, journal->tail + 1);  in advance_tail()
975 	vdo_advance_block_map_era(journal->block_map, journal->tail);  in advance_tail()
985 static void initialize_lock_count(struct recovery_journal *journal)  in initialize_lock_count()  argument
988 	block_count_t lock_number = journal->active_block->block_number;  in initialize_lock_count()
989 	atomic_t *decrement_counter = get_decrement_counter(journal, lock_number);  in initialize_lock_count()
991 	journal_value = get_counter(journal, lock_number, VDO_ZONE_TYPE_JOURNAL, 0);  in initialize_lock_count()
994 	*journal_value = journal->entries_per_block + 1;  in initialize_lock_count()
1005 static bool prepare_to_assign_entry(struct recovery_journal *journal)  in prepare_to_assign_entry()  argument
1007 	if (journal->available_space == 0)  in prepare_to_assign_entry()
1010 	if (is_block_full(journal->active_block) && !advance_tail(journal))  in prepare_to_assign_entry()
1013 	if (!is_block_empty(journal->active_block))  in prepare_to_assign_entry()
1016 	if ((journal->tail - get_recovery_journal_head(journal)) > journal->size) {  in prepare_to_assign_entry()
1018 		journal->events.disk_full++;  in prepare_to_assign_entry()
1028 	initialize_lock_count(journal);  in prepare_to_assign_entry()
1032 static void write_blocks(struct recovery_journal *journal);
1043 static void schedule_block_write(struct recovery_journal *journal,  in schedule_block_write()  argument
1047 		vdo_waitq_enqueue_waiter(&journal->pending_writes, &block->write_waiter);  in schedule_block_write()
1060 	vdo_release_recovery_journal_block_reference(block->journal,  in release_journal_block_reference()
1065 static void update_usages(struct recovery_journal *journal, struct data_vio *data_vio)  in update_usages()  argument
1068 		journal->block_map_data_blocks++;  in update_usages()
1073 		journal->logical_blocks_used++;  in update_usages()
1076 		journal->logical_blocks_used--;  in update_usages()
1088 	struct recovery_journal *journal = block->journal;  in assign_entry()  local
1096 	update_usages(journal, data_vio);  in assign_entry()
1097 	journal->available_space--;  in assign_entry()
1100 		journal->events.blocks.started++;  in assign_entry()
1105 	journal->events.entries.started++;  in assign_entry()
1112 		schedule_block_write(journal, block);  in assign_entry()
1116 	check_slab_journal_commit_threshold(journal);  in assign_entry()
1119 static void assign_entries(struct recovery_journal *journal)  in assign_entries()  argument
1121 	if (journal->adding_entries) {  in assign_entries()
1126 	journal->adding_entries = true;  in assign_entries()
1127 	while (vdo_waitq_has_waiters(&journal->entry_waiters) &&  in assign_entries()
1128 	       prepare_to_assign_entry(journal)) {  in assign_entries()
1129 		vdo_waitq_notify_next_waiter(&journal->entry_waiters,  in assign_entries()
1130 					     assign_entry, journal->active_block);  in assign_entries()
1134 	write_blocks(journal);  in assign_entries()
1135 	journal->adding_entries = false;  in assign_entries()
1145 	struct recovery_journal *journal = block->journal;  in recycle_journal_block()  local
1148 	list_move_tail(&block->list_node, &journal->free_tail_blocks);  in recycle_journal_block()
1151 	for (i = block->entry_count; i < journal->entries_per_block; i++)  in recycle_journal_block()
1161 	if (block == journal->active_block)  in recycle_journal_block()
1162 		journal->active_block = NULL;  in recycle_journal_block()
1174 	struct recovery_journal *journal = context;  in continue_committed_waiter()  local
1175 	int result = (is_read_only(journal) ? VDO_READ_ONLY : VDO_SUCCESS);  in continue_committed_waiter()
1178 	VDO_ASSERT_LOG_ONLY(vdo_before_journal_point(&journal->commit_point,  in continue_committed_waiter()
1181 			    (unsigned long long) journal->commit_point.sequence_number,  in continue_committed_waiter()
1182 			    journal->commit_point.entry_count,  in continue_committed_waiter()
1186 	journal->commit_point = data_vio->recovery_journal_point;  in continue_committed_waiter()
1209 static void notify_commit_waiters(struct recovery_journal *journal)  in notify_commit_waiters()  argument
1213 	list_for_each_entry(block, &journal->active_tail_blocks, list_node) {  in notify_commit_waiters()
1218 					     continue_committed_waiter, journal);  in notify_commit_waiters()
1219 		if (is_read_only(journal)) {  in notify_commit_waiters()
1222 						     journal);  in notify_commit_waiters()
1234 static void recycle_journal_blocks(struct recovery_journal *journal)  in recycle_journal_blocks()  argument
1238 	list_for_each_entry_safe(block, tmp, &journal->active_tail_blocks, list_node) {  in recycle_journal_blocks()
1244 		if (!is_read_only(journal) &&  in recycle_journal_blocks()
1267 	struct recovery_journal *journal = block->journal;  in complete_write()  local
1270 	assert_on_journal_thread(journal, __func__);  in complete_write()
1272 	journal->pending_write_count -= 1;  in complete_write()
1273 	journal->events.blocks.committed += 1;  in complete_write()
1274 	journal->events.entries.committed += block->entries_in_commit;  in complete_write()
1280 	if (block->sequence_number > journal->last_write_acknowledged)  in complete_write()
1281 		journal->last_write_acknowledged = block->sequence_number;  in complete_write()
1283 	last_active_block = get_journal_block(&journal->active_tail_blocks);  in complete_write()
1287 	notify_commit_waiters(journal);  in complete_write()
1294 		schedule_block_write(journal, block);  in complete_write()
1296 	recycle_journal_blocks(journal);  in complete_write()
1297 	write_blocks(journal);  in complete_write()
1299 	check_for_drain_complete(journal);  in complete_write()
1305 	struct recovery_journal *journal = block->journal;  in handle_write_error()  local
1311 	enter_journal_read_only_mode(journal, completion->result);  in handle_write_error()
1319 	struct recovery_journal *journal = block->journal;  in complete_write_endio()  local
1321 	continue_vio_after_io(vio, complete_write, journal->thread_id);  in complete_write_endio()
1372 	struct recovery_journal *journal = block->journal;  in write_block()  local
1376 	    is_read_only(journal))  in write_block()
1382 	journal->pending_write_count += 1;  in write_block()
1383 	journal->events.blocks.written += 1;  in write_block()
1384 	journal->events.entries.written += block->entries_in_commit;  in write_block()
1386 	header->block_map_head = __cpu_to_le64(journal->block_map_head);  in write_block()
1387 	header->slab_journal_head = __cpu_to_le64(journal->slab_journal_head);  in write_block()
1397 	vdo_submit_metadata_vio(&block->vio, journal->origin + block->block_number,  in write_block()
1407 static void write_blocks(struct recovery_journal *journal)  in write_blocks()  argument
1409 	assert_on_journal_thread(journal, __func__);  in write_blocks()
1419 	if (journal->pending_write_count > 0)  in write_blocks()
1423 	vdo_waitq_notify_all_waiters(&journal->pending_writes, write_block, NULL);  in write_blocks()
1429 	if ((journal->pending_write_count == 0) && (journal->active_block != NULL))  in write_blocks()
1430 		write_block(&journal->active_block->write_waiter, NULL);  in write_blocks()
1445 void vdo_add_recovery_journal_entry(struct recovery_journal *journal,  in vdo_add_recovery_journal_entry()  argument
1448 	assert_on_journal_thread(journal, __func__);  in vdo_add_recovery_journal_entry()
1449 	if (!vdo_is_state_normal(&journal->state)) {  in vdo_add_recovery_journal_entry()
1454 	if (is_read_only(journal)) {  in vdo_add_recovery_journal_entry()
1462 	vdo_advance_journal_point(&journal->append_point, journal->entries_per_block);  in vdo_add_recovery_journal_entry()
1463 	vdo_waitq_enqueue_waiter(&journal->entry_waiters, &data_vio->waiter);  in vdo_add_recovery_journal_entry()
1464 	assign_entries(journal);  in vdo_add_recovery_journal_entry()
1478 static bool is_lock_locked(struct recovery_journal *journal, block_count_t lock_number,  in is_lock_locked()  argument
1484 	if (is_journal_zone_locked(journal, lock_number))  in is_lock_locked()
1487 	zone_count = get_zone_count_ptr(journal, lock_number, zone_type);  in is_lock_locked()
1498 static void reap_recovery_journal(struct recovery_journal *journal)  in reap_recovery_journal()  argument
1500 	if (journal->reaping) {  in reap_recovery_journal()
1508 	if (vdo_is_state_quiescent(&journal->state)) {  in reap_recovery_journal()
1517 	while ((journal->block_map_reap_head < journal->last_write_acknowledged) &&  in reap_recovery_journal()
1518 		!is_lock_locked(journal, journal->block_map_head_block_number,  in reap_recovery_journal()
1520 		journal->block_map_reap_head++;  in reap_recovery_journal()
1521 		if (++journal->block_map_head_block_number == journal->size)  in reap_recovery_journal()
1522 			journal->block_map_head_block_number = 0;  in reap_recovery_journal()
1525 	while ((journal->slab_journal_reap_head < journal->last_write_acknowledged) &&  in reap_recovery_journal()
1526 		!is_lock_locked(journal, journal->slab_journal_head_block_number,  in reap_recovery_journal()
1528 		journal->slab_journal_reap_head++;  in reap_recovery_journal()
1529 		if (++journal->slab_journal_head_block_number == journal->size)  in reap_recovery_journal()
1530 			journal->slab_journal_head_block_number = 0;  in reap_recovery_journal()
1533 	if ((journal->block_map_reap_head == journal->block_map_head) &&  in reap_recovery_journal()
1534 	    (journal->slab_journal_reap_head == journal->slab_journal_head)) {  in reap_recovery_journal()
1544 	journal->reaping = true;  in reap_recovery_journal()
1545 	vdo_submit_flush_vio(journal->flush_vio, flush_endio, handle_flush_error);  in reap_recovery_journal()
1556 void vdo_acquire_recovery_journal_block_reference(struct recovery_journal *journal,  in vdo_acquire_recovery_journal_block_reference()  argument
1570 	lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number);  in vdo_acquire_recovery_journal_block_reference()
1571 	current_value = get_counter(journal, lock_number, zone_type, zone_id);  in vdo_acquire_recovery_journal_block_reference()
1581 		atomic_inc(get_zone_count_ptr(journal, lock_number, zone_type));  in vdo_acquire_recovery_journal_block_reference()
1595 void vdo_release_journal_entry_lock(struct recovery_journal *journal,  in vdo_release_journal_entry_lock()  argument
1603 	lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number);  in vdo_release_journal_entry_lock()
1609 	atomic_inc(get_decrement_counter(journal, lock_number));  in vdo_release_journal_entry_lock()
1632 void vdo_drain_recovery_journal(struct recovery_journal *journal,  in vdo_drain_recovery_journal()  argument
1636 	assert_on_journal_thread(journal, __func__);  in vdo_drain_recovery_journal()
1637 	vdo_start_draining(&journal->state, operation, parent, initiate_drain);  in vdo_drain_recovery_journal()
1668 void vdo_resume_recovery_journal(struct recovery_journal *journal,  in vdo_resume_recovery_journal()  argument
1673 	assert_on_journal_thread(journal, __func__);  in vdo_resume_recovery_journal()
1674 	saved = vdo_is_state_saved(&journal->state);  in vdo_resume_recovery_journal()
1675 	vdo_set_completion_result(parent, vdo_resume_if_quiescent(&journal->state));  in vdo_resume_recovery_journal()
1676 	if (is_read_only(journal)) {  in vdo_resume_recovery_journal()
1682 		initialize_journal_state(journal);  in vdo_resume_recovery_journal()
1684 	if (resume_lock_counter(&journal->lock_counter)) {  in vdo_resume_recovery_journal()
1686 		reap_recovery_journal(journal);  in vdo_resume_recovery_journal()
1699 block_count_t vdo_get_recovery_journal_logical_blocks_used(const struct recovery_journal *journal)  in vdo_get_recovery_journal_logical_blocks_used()  argument
1701 	return journal->logical_blocks_used;  in vdo_get_recovery_journal_logical_blocks_used()
1711 vdo_get_recovery_journal_statistics(const struct recovery_journal *journal)  in vdo_get_recovery_journal_statistics()  argument
1713 	return journal->events;  in vdo_get_recovery_journal_statistics()
1734 void vdo_dump_recovery_journal_statistics(const struct recovery_journal *journal)  in vdo_dump_recovery_journal_statistics()  argument
1737 	struct recovery_journal_statistics stats = vdo_get_recovery_journal_statistics(journal);  in vdo_dump_recovery_journal_statistics()
1741 		     (unsigned long long) journal->block_map_head,  in vdo_dump_recovery_journal_statistics()
1742 		     (unsigned long long) journal->slab_journal_head,  in vdo_dump_recovery_journal_statistics()
1743 		     (unsigned long long) journal->last_write_acknowledged,  in vdo_dump_recovery_journal_statistics()
1744 		     (unsigned long long) journal->tail,  in vdo_dump_recovery_journal_statistics()
1745 		     (unsigned long long) journal->block_map_reap_head,  in vdo_dump_recovery_journal_statistics()
1746 		     (unsigned long long) journal->slab_journal_reap_head,  in vdo_dump_recovery_journal_statistics()
1749 		     vdo_waitq_num_waiters(&journal->entry_waiters));  in vdo_dump_recovery_journal_statistics()
1760 	list_for_each_entry(block, &journal->active_tail_blocks, list_node)  in vdo_dump_recovery_journal_statistics()