Lines Matching +full:mode +full:- +full:recovery
1 // SPDX-License-Identifier: GPL-2.0-only
6 #include "recovery-journal.h"
12 #include "memory-alloc.h"
15 #include "block-map.h"
18 #include "data-vio.h"
20 #include "io-submitter.h"
21 #include "slab-depot.h"
25 #include "wait-queue.h"
30 * The number of reserved blocks must be large enough to prevent a new recovery journal
40 * A lock_counter is intended to keep all of the locks for the blocks in the recovery journal. The
41 * per-zone counters are all kept in a single array which is arranged by zone (i.e. zone 0's lock 0
43 * arrangement is intended to minimize cache-line contention for counters from different zones.
49 * Lock sets are laid out with the set for recovery journal first, followed by the logical zones,
60 * get_zone_count_ptr() - Get a pointer to the zone count for a given lock on a given zone.
61 * @journal: The recovery journal.
72 ? &journal->lock_counter.logical_zone_counts[lock_number] in get_zone_count_ptr()
73 : &journal->lock_counter.physical_zone_counts[lock_number]); in get_zone_count_ptr()
77 * get_counter() - Get the zone counter for a given lock on a given zone.
78 * @journal: The recovery journal.
89 struct lock_counter *counter = &journal->lock_counter; in get_counter()
90 block_count_t zone_counter = (counter->locks * zone_id) + lock_number; in get_counter()
93 return &counter->journal_counters[zone_counter]; in get_counter()
96 return &counter->logical_counters[zone_counter]; in get_counter()
98 return &counter->physical_counters[zone_counter]; in get_counter()
104 return &journal->lock_counter.journal_decrement_counts[lock_number]; in get_decrement_counter()
108 * is_journal_zone_locked() - Check whether the journal zone is locked for a given lock.
109 * @journal: The recovery journal.
128 * vdo_release_recovery_journal_block_reference() - Release a reference to a recovery journal
130 * @journal: The recovery journal.
155 *current_value -= 1; in vdo_release_recovery_journal_block_reference()
168 if (atomic_add_return(-1, zone_count) > 0) in vdo_release_recovery_journal_block_reference()
177 prior_state = atomic_cmpxchg(&journal->lock_counter.state, in vdo_release_recovery_journal_block_reference()
186 vdo_launch_completion(&journal->lock_counter.completion); in vdo_release_recovery_journal_block_reference()
195 * pop_free_list() - Get a block from the end of the free list.
204 if (list_empty(&journal->free_tail_blocks)) in pop_free_list()
207 block = list_last_entry(&journal->free_tail_blocks, in pop_free_list()
209 list_del_init(&block->list_node); in pop_free_list()
214 * is_block_dirty() - Check whether a recovery block is dirty.
224 return (block->uncommitted_entry_count > 0); in is_block_dirty()
228 * is_block_empty() - Check whether a journal block is empty.
235 return (block->entry_count == 0); in is_block_empty()
239 * is_block_full() - Check whether a journal block is full.
246 return ((block == NULL) || (block->journal->entries_per_block == block->entry_count)); in is_block_full()
250 * assert_on_journal_thread() - Assert that we are running on the journal thread.
257 VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == journal->thread_id), in assert_on_journal_thread()
262 * continue_waiter() - Release a data_vio from the journal.
273 * has_block_waiters() - Check whether the journal has any waiters on any blocks.
280 struct recovery_journal_block *block = get_journal_block(&journal->active_tail_blocks); in has_block_waiters()
287 (vdo_waitq_has_waiters(&block->entry_waiters) || in has_block_waiters()
288 vdo_waitq_has_waiters(&block->commit_waiters))); in has_block_waiters()
296 * suspend_lock_counter() - Prevent the lock counter from notifying.
310 prior_state = atomic_cmpxchg(&counter->state, LOCK_COUNTER_STATE_NOT_NOTIFYING, in suspend_lock_counter()
321 return vdo_is_read_only(journal->flush_vio->completion.vdo); in is_read_only()
325 * check_for_drain_complete() - Check whether the journal has drained.
335 * Clean up any full active blocks which were not written due to read-only mode. in check_for_drain_complete()
337 * FIXME: This would probably be better as a short-circuit in write_block(). in check_for_drain_complete()
343 vdo_waitq_notify_all_waiters(&journal->entry_waiters, in check_for_drain_complete()
347 if (!vdo_is_state_draining(&journal->state) || in check_for_drain_complete()
348 journal->reaping || in check_for_drain_complete()
350 vdo_waitq_has_waiters(&journal->entry_waiters) || in check_for_drain_complete()
351 !suspend_lock_counter(&journal->lock_counter)) in check_for_drain_complete()
354 if (vdo_is_state_saving(&journal->state)) { in check_for_drain_complete()
355 if (journal->active_block != NULL) { in check_for_drain_complete()
357 !is_block_dirty(journal->active_block)), in check_for_drain_complete()
359 recycle_journal_block(journal->active_block); in check_for_drain_complete()
362 VDO_ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks), in check_for_drain_complete()
366 vdo_finish_draining_with_result(&journal->state, result); in check_for_drain_complete()
370 * notify_recovery_journal_of_read_only_mode() - Notify a recovery journal that the VDO has gone
371 * read-only.
385 * enter_journal_read_only_mode() - Put the journal in read-only mode.
395 vdo_enter_read_only_mode(journal->flush_vio->completion.vdo, error_code); in enter_journal_read_only_mode()
400 * vdo_get_recovery_journal_current_sequence_number() - Obtain the recovery journal's current
410 return journal->tail; in vdo_get_recovery_journal_current_sequence_number()
414 * get_recovery_journal_head() - Get the head of the recovery journal.
423 return min(journal->block_map_head, journal->slab_journal_head); in get_recovery_journal_head()
427 * compute_recovery_count_byte() - Compute the recovery count byte for a given recovery count.
428 * @recovery_count: The recovery count.
430 * Return: The byte corresponding to the recovery count.
438 * check_slab_journal_commit_threshold() - Check whether the journal is over the threshold, and if
444 block_count_t current_length = journal->tail - journal->slab_journal_head; in check_slab_journal_commit_threshold()
446 if (current_length > journal->slab_journal_commit_threshold) { in check_slab_journal_commit_threshold()
447 journal->events.slab_journal_commits_requested++; in check_slab_journal_commit_threshold()
448 vdo_commit_oldest_slab_journal_tail_blocks(journal->depot, in check_slab_journal_commit_threshold()
449 journal->slab_journal_head); in check_slab_journal_commit_threshold()
457 * finish_reaping() - Finish reaping the journal.
465 journal->block_map_head = journal->block_map_reap_head; in finish_reaping()
466 journal->slab_journal_head = journal->slab_journal_reap_head; in finish_reaping()
467 blocks_reaped = get_recovery_journal_head(journal) - old_head; in finish_reaping()
468 journal->available_space += blocks_reaped * journal->entries_per_block; in finish_reaping()
469 journal->reaping = false; in finish_reaping()
476 * complete_reaping() - Finish reaping the journal after flushing the lower layer.
483 struct recovery_journal *journal = completion->parent; in complete_reaping()
492 * handle_flush_error() - Handle an error when flushing the lower layer due to reaping.
497 struct recovery_journal *journal = completion->parent; in handle_flush_error()
500 journal->reaping = false; in handle_flush_error()
501 enter_journal_read_only_mode(journal, completion->result); in handle_flush_error()
506 struct vio *vio = bio->bi_private; in flush_endio()
507 struct recovery_journal *journal = vio->completion.parent; in flush_endio()
509 continue_vio_after_io(vio, complete_reaping, journal->thread_id); in flush_endio()
513 * initialize_journal_state() - Set all journal fields appropriately to start journaling from the
519 journal->append_point.sequence_number = journal->tail; in initialize_journal_state()
520 journal->last_write_acknowledged = journal->tail; in initialize_journal_state()
521 journal->block_map_head = journal->tail; in initialize_journal_state()
522 journal->slab_journal_head = journal->tail; in initialize_journal_state()
523 journal->block_map_reap_head = journal->tail; in initialize_journal_state()
524 journal->slab_journal_reap_head = journal->tail; in initialize_journal_state()
525 journal->block_map_head_block_number = in initialize_journal_state()
526 vdo_get_recovery_journal_block_number(journal, journal->block_map_head); in initialize_journal_state()
527 journal->slab_journal_head_block_number = in initialize_journal_state()
529 journal->slab_journal_head); in initialize_journal_state()
530 journal->available_space = in initialize_journal_state()
531 (journal->entries_per_block * vdo_get_recovery_journal_length(journal->size)); in initialize_journal_state()
535 * vdo_get_recovery_journal_length() - Get the number of usable recovery journal blocks.
536 * @journal_size: The size of the recovery journal in blocks.
538 * Return: the number of recovery journal blocks usable for entries.
546 return (journal_size - reserved_blocks); in vdo_get_recovery_journal_length()
550 * reap_recovery_journal_callback() - Attempt to reap the journal.
558 struct recovery_journal *journal = (struct recovery_journal *) completion->parent; in reap_recovery_journal_callback()
564 atomic_set(&journal->lock_counter.state, LOCK_COUNTER_STATE_NOT_NOTIFYING); in reap_recovery_journal_callback()
566 if (vdo_is_state_quiescing(&journal->state)) { in reap_recovery_journal_callback()
580 * initialize_lock_counter() - Initialize a lock counter.
582 * @journal: The recovery journal.
591 struct thread_config *config = &vdo->thread_config; in initialize_lock_counter()
592 struct lock_counter *counter = &journal->lock_counter; in initialize_lock_counter()
594 result = vdo_allocate(journal->size, u16, __func__, &counter->journal_counters); in initialize_lock_counter()
598 result = vdo_allocate(journal->size, atomic_t, __func__, in initialize_lock_counter()
599 &counter->journal_decrement_counts); in initialize_lock_counter()
603 result = vdo_allocate(journal->size * config->logical_zone_count, u16, __func__, in initialize_lock_counter()
604 &counter->logical_counters); in initialize_lock_counter()
608 result = vdo_allocate(journal->size, atomic_t, __func__, in initialize_lock_counter()
609 &counter->logical_zone_counts); in initialize_lock_counter()
613 result = vdo_allocate(journal->size * config->physical_zone_count, u16, __func__, in initialize_lock_counter()
614 &counter->physical_counters); in initialize_lock_counter()
618 result = vdo_allocate(journal->size, atomic_t, __func__, in initialize_lock_counter()
619 &counter->physical_zone_counts); in initialize_lock_counter()
623 vdo_initialize_completion(&counter->completion, vdo, in initialize_lock_counter()
625 vdo_prepare_completion(&counter->completion, reap_recovery_journal_callback, in initialize_lock_counter()
626 reap_recovery_journal_callback, config->journal_thread, in initialize_lock_counter()
628 counter->logical_zones = config->logical_zone_count; in initialize_lock_counter()
629 counter->physical_zones = config->physical_zone_count; in initialize_lock_counter()
630 counter->locks = journal->size; in initialize_lock_counter()
635 * set_journal_tail() - Set the journal's tail sequence number.
645 journal->tail = tail; in set_journal_tail()
649 * initialize_recovery_block() - Initialize a journal block.
666 ((VDO_BLOCK_SIZE - sizeof(struct packed_journal_header)) / in initialize_recovery_block()
678 VIO_PRIORITY_HIGH, block, 1, data, &block->vio); in initialize_recovery_block()
684 list_add_tail(&block->list_node, &journal->free_tail_blocks); in initialize_recovery_block()
685 block->journal = journal; in initialize_recovery_block()
690 * vdo_decode_recovery_journal() - Make a recovery journal and initialize it with the state that
699 * @journal_ptr: The pointer to hold the new recovery journal.
719 INIT_LIST_HEAD(&journal->free_tail_blocks); in vdo_decode_recovery_journal()
720 INIT_LIST_HEAD(&journal->active_tail_blocks); in vdo_decode_recovery_journal()
721 vdo_waitq_init(&journal->pending_writes); in vdo_decode_recovery_journal()
723 journal->thread_id = vdo->thread_config.journal_thread; in vdo_decode_recovery_journal()
724 journal->origin = partition->offset; in vdo_decode_recovery_journal()
725 journal->nonce = nonce; in vdo_decode_recovery_journal()
726 journal->recovery_count = compute_recovery_count_byte(recovery_count); in vdo_decode_recovery_journal()
727 journal->size = journal_size; in vdo_decode_recovery_journal()
728 journal->slab_journal_commit_threshold = (journal_size * 2) / 3; in vdo_decode_recovery_journal()
729 journal->logical_blocks_used = state.logical_blocks_used; in vdo_decode_recovery_journal()
730 journal->block_map_data_blocks = state.block_map_data_blocks; in vdo_decode_recovery_journal()
731 journal->entries_per_block = RECOVERY_JOURNAL_ENTRIES_PER_BLOCK; in vdo_decode_recovery_journal()
735 vdo_set_admin_state_code(&journal->state, VDO_ADMIN_STATE_SUSPENDED); in vdo_decode_recovery_journal()
738 struct recovery_journal_block *block = &journal->blocks[i]; in vdo_decode_recovery_journal()
754 journal, NULL, &journal->flush_vio); in vdo_decode_recovery_journal()
762 journal->thread_id); in vdo_decode_recovery_journal()
768 result = vdo_make_default_thread(vdo, journal->thread_id); in vdo_decode_recovery_journal()
774 journal->flush_vio->completion.callback_thread_id = journal->thread_id; in vdo_decode_recovery_journal()
780 * vdo_free_recovery_journal() - Free a recovery journal.
781 * @journal: The recovery journal to free.
790 vdo_free(vdo_forget(journal->lock_counter.logical_zone_counts)); in vdo_free_recovery_journal()
791 vdo_free(vdo_forget(journal->lock_counter.physical_zone_counts)); in vdo_free_recovery_journal()
792 vdo_free(vdo_forget(journal->lock_counter.journal_counters)); in vdo_free_recovery_journal()
793 vdo_free(vdo_forget(journal->lock_counter.journal_decrement_counts)); in vdo_free_recovery_journal()
794 vdo_free(vdo_forget(journal->lock_counter.logical_counters)); in vdo_free_recovery_journal()
795 vdo_free(vdo_forget(journal->lock_counter.physical_counters)); in vdo_free_recovery_journal()
796 free_vio(vdo_forget(journal->flush_vio)); in vdo_free_recovery_journal()
802 if (!vdo_is_state_quiescent(&journal->state)) { in vdo_free_recovery_journal()
803 VDO_ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks), in vdo_free_recovery_journal()
805 } else if (!vdo_is_state_saved(&journal->state) && in vdo_free_recovery_journal()
806 !list_empty(&journal->active_tail_blocks)) { in vdo_free_recovery_journal()
811 struct recovery_journal_block *block = &journal->blocks[i]; in vdo_free_recovery_journal()
813 vdo_free(vdo_forget(block->vio.data)); in vdo_free_recovery_journal()
814 free_vio_components(&block->vio); in vdo_free_recovery_journal()
821 * vdo_initialize_recovery_journal_post_repair() - Initialize the journal after a repair.
835 journal->recovery_count = compute_recovery_count_byte(recovery_count); in vdo_initialize_recovery_journal_post_repair()
837 journal->logical_blocks_used = logical_blocks_used; in vdo_initialize_recovery_journal_post_repair()
838 journal->block_map_data_blocks = block_map_data_blocks; in vdo_initialize_recovery_journal_post_repair()
842 * vdo_get_journal_block_map_data_blocks_used() - Get the number of block map pages, allocated from
850 return journal->block_map_data_blocks; in vdo_get_journal_block_map_data_blocks_used()
854 * vdo_get_recovery_journal_thread_id() - Get the ID of a recovery journal's thread.
861 return journal->thread_id; in vdo_get_recovery_journal_thread_id()
865 * vdo_open_recovery_journal() - Prepare the journal for new entries.
873 journal->depot = depot; in vdo_open_recovery_journal()
874 journal->block_map = block_map; in vdo_open_recovery_journal()
875 WRITE_ONCE(journal->state.current_state, VDO_ADMIN_STATE_NORMAL_OPERATION); in vdo_open_recovery_journal()
879 * vdo_record_recovery_journal() - Record the state of a recovery journal for encoding in the super
881 * @journal: the recovery journal.
889 .logical_blocks_used = journal->logical_blocks_used, in vdo_record_recovery_journal()
890 .block_map_data_blocks = journal->block_map_data_blocks, in vdo_record_recovery_journal()
893 if (vdo_is_state_saved(&journal->state)) { in vdo_record_recovery_journal()
898 state.journal_start = journal->tail; in vdo_record_recovery_journal()
901 * When we're merely suspended or have gone read-only, we must record the first in vdo_record_recovery_journal()
911 * get_block_header() - Get a pointer to the packed journal block header in the block buffer.
912 * @block: The recovery block.
919 return (struct packed_journal_header *) block->vio.data; in get_block_header()
923 * set_active_sector() - Set the current sector of the current block and initialize it.
929 block->sector = sector; in set_active_sector()
930 block->sector->check_byte = get_block_header(block)->check_byte; in set_active_sector()
931 block->sector->recovery_count = block->journal->recovery_count; in set_active_sector()
932 block->sector->entry_count = 0; in set_active_sector()
936 * advance_tail() - Advance the tail of the journal.
947 block = journal->active_block = pop_free_list(journal); in advance_tail()
951 list_move_tail(&block->list_node, &journal->active_tail_blocks); in advance_tail()
955 .block_map_data_blocks = journal->block_map_data_blocks, in advance_tail()
956 .logical_blocks_used = journal->logical_blocks_used, in advance_tail()
957 .nonce = journal->nonce, in advance_tail()
958 .recovery_count = journal->recovery_count, in advance_tail()
959 .sequence_number = journal->tail, in advance_tail()
961 journal->tail), in advance_tail()
965 memset(block->vio.data, 0x0, VDO_BLOCK_SIZE); in advance_tail()
966 block->sequence_number = journal->tail; in advance_tail()
967 block->entry_count = 0; in advance_tail()
968 block->uncommitted_entry_count = 0; in advance_tail()
969 block->block_number = vdo_get_recovery_journal_block_number(journal, in advance_tail()
970 journal->tail); in advance_tail()
974 set_journal_tail(journal, journal->tail + 1); in advance_tail()
975 vdo_advance_block_map_era(journal->block_map, journal->tail); in advance_tail()
980 * initialize_lock_count() - Initialize the value of the journal zone's counter for a given lock.
981 * @journal: The recovery journal.
988 block_count_t lock_number = journal->active_block->block_number; in initialize_lock_count()
994 *journal_value = journal->entries_per_block + 1; in initialize_lock_count()
999 * prepare_to_assign_entry() - Prepare the currently active block to receive an entry and check
1007 if (journal->available_space == 0) in prepare_to_assign_entry()
1010 if (is_block_full(journal->active_block) && !advance_tail(journal)) in prepare_to_assign_entry()
1013 if (!is_block_empty(journal->active_block)) in prepare_to_assign_entry()
1016 if ((journal->tail - get_recovery_journal_head(journal)) > journal->size) { in prepare_to_assign_entry()
1018 journal->events.disk_full++; in prepare_to_assign_entry()
1026 * per-entry lock for the block map entry serves to protect those as well. in prepare_to_assign_entry()
1035 * schedule_block_write() - Queue a block for writing.
1046 if (!block->committing) in schedule_block_write()
1047 vdo_waitq_enqueue_waiter(&journal->pending_writes, &block->write_waiter); in schedule_block_write()
1055 * release_journal_block_reference() - Release a reference to a journal block.
1060 vdo_release_recovery_journal_block_reference(block->journal, in release_journal_block_reference()
1061 block->sequence_number, in release_journal_block_reference()
1067 if (data_vio->increment_updater.operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) { in update_usages()
1068 journal->block_map_data_blocks++; in update_usages()
1072 if (data_vio->new_mapped.state != VDO_MAPPING_STATE_UNMAPPED) in update_usages()
1073 journal->logical_blocks_used++; in update_usages()
1075 if (data_vio->mapped.state != VDO_MAPPING_STATE_UNMAPPED) in update_usages()
1076 journal->logical_blocks_used--; in update_usages()
1080 * assign_entry() - Assign an entry waiter to the active block.
1088 struct recovery_journal *journal = block->journal; in assign_entry()
1091 data_vio->recovery_journal_point = (struct journal_point) { in assign_entry()
1092 .sequence_number = block->sequence_number, in assign_entry()
1093 .entry_count = block->entry_count, in assign_entry()
1097 journal->available_space--; in assign_entry()
1099 if (!vdo_waitq_has_waiters(&block->entry_waiters)) in assign_entry()
1100 journal->events.blocks.started++; in assign_entry()
1102 vdo_waitq_enqueue_waiter(&block->entry_waiters, &data_vio->waiter); in assign_entry()
1103 block->entry_count++; in assign_entry()
1104 block->uncommitted_entry_count++; in assign_entry()
1105 journal->events.entries.started++; in assign_entry()
1121 if (journal->adding_entries) { in assign_entries()
1122 /* Protect against re-entrancy. */ in assign_entries()
1126 journal->adding_entries = true; in assign_entries()
1127 while (vdo_waitq_has_waiters(&journal->entry_waiters) && in assign_entries()
1129 vdo_waitq_notify_next_waiter(&journal->entry_waiters, in assign_entries()
1130 assign_entry, journal->active_block); in assign_entries()
1135 journal->adding_entries = false; in assign_entries()
1139 * recycle_journal_block() - Prepare an in-memory journal block to be reused now that it has been
1145 struct recovery_journal *journal = block->journal; in recycle_journal_block()
1148 list_move_tail(&block->list_node, &journal->free_tail_blocks); in recycle_journal_block()
1151 for (i = block->entry_count; i < journal->entries_per_block; i++) in recycle_journal_block()
1156 * we're giving up because we're in read-only mode. in recycle_journal_block()
1158 if (block->entry_count > 0) in recycle_journal_block()
1161 if (block == journal->active_block) in recycle_journal_block()
1162 journal->active_block = NULL; in recycle_journal_block()
1166 * continue_committed_waiter() - invoked whenever a VIO is to be released from the journal because
1178 VDO_ASSERT_LOG_ONLY(vdo_before_journal_point(&journal->commit_point, in continue_committed_waiter()
1179 &data_vio->recovery_journal_point), in continue_committed_waiter()
1180 …"DataVIOs released from recovery journal in order. Recovery journal point is (%llu, %u), but commi… in continue_committed_waiter()
1181 (unsigned long long) journal->commit_point.sequence_number, in continue_committed_waiter()
1182 journal->commit_point.entry_count, in continue_committed_waiter()
1183 (unsigned long long) data_vio->recovery_journal_point.sequence_number, in continue_committed_waiter()
1184 data_vio->recovery_journal_point.entry_count); in continue_committed_waiter()
1186 journal->commit_point = data_vio->recovery_journal_point; in continue_committed_waiter()
1187 data_vio->last_async_operation = VIO_ASYNC_OP_UPDATE_REFERENCE_COUNTS; in continue_committed_waiter()
1197 has_decrement = (data_vio->decrement_updater.zpbn.pbn != VDO_ZERO_BLOCK); in continue_committed_waiter()
1198 if ((data_vio->increment_updater.zpbn.pbn != VDO_ZERO_BLOCK) || !has_decrement) in continue_committed_waiter()
1202 vdo_launch_completion(&data_vio->decrement_completion); in continue_committed_waiter()
1206 * notify_commit_waiters() - Notify any VIOs whose entries have now committed.
1207 * @journal: The recovery journal to update.
1213 list_for_each_entry(block, &journal->active_tail_blocks, list_node) { in notify_commit_waiters()
1214 if (block->committing) in notify_commit_waiters()
1217 vdo_waitq_notify_all_waiters(&block->commit_waiters, in notify_commit_waiters()
1220 vdo_waitq_notify_all_waiters(&block->entry_waiters, in notify_commit_waiters()
1224 /* Stop at partially-committed or partially-filled blocks. */ in notify_commit_waiters()
1231 * recycle_journal_blocks() - Recycle any journal blocks which have been fully committed.
1232 * @journal: The recovery journal to update.
1238 list_for_each_entry_safe(block, tmp, &journal->active_tail_blocks, list_node) { in recycle_journal_blocks()
1239 if (block->committing) { in recycle_journal_blocks()
1248 * read-only mode. in recycle_journal_blocks()
1258 * complete_write() - Handle post-commit processing.
1266 struct recovery_journal_block *block = completion->parent; in complete_write()
1267 struct recovery_journal *journal = block->journal; in complete_write()
1272 journal->pending_write_count -= 1; in complete_write()
1273 journal->events.blocks.committed += 1; in complete_write()
1274 journal->events.entries.committed += block->entries_in_commit; in complete_write()
1275 block->uncommitted_entry_count -= block->entries_in_commit; in complete_write()
1276 block->entries_in_commit = 0; in complete_write()
1277 block->committing = false; in complete_write()
1280 if (block->sequence_number > journal->last_write_acknowledged) in complete_write()
1281 journal->last_write_acknowledged = block->sequence_number; in complete_write()
1283 last_active_block = get_journal_block(&journal->active_tail_blocks); in complete_write()
1284 VDO_ASSERT_LOG_ONLY((block->sequence_number >= last_active_block->sequence_number), in complete_write()
1304 struct recovery_journal_block *block = completion->parent; in handle_write_error()
1305 struct recovery_journal *journal = block->journal; in handle_write_error()
1308 vdo_log_error_strerror(completion->result, in handle_write_error()
1309 "cannot write recovery journal block %llu", in handle_write_error()
1310 (unsigned long long) block->sequence_number); in handle_write_error()
1311 enter_journal_read_only_mode(journal, completion->result); in handle_write_error()
1317 struct vio *vio = bio->bi_private; in complete_write_endio()
1318 struct recovery_journal_block *block = vio->completion.parent; in complete_write_endio()
1319 struct recovery_journal *journal = block->journal; in complete_write_endio()
1321 continue_vio_after_io(vio, complete_write, journal->thread_id); in complete_write_endio()
1325 * add_queued_recovery_entries() - Actually add entries from the queue to the given block.
1330 while (vdo_waitq_has_waiters(&block->entry_waiters)) { in add_queued_recovery_entries()
1332 vdo_waiter_as_data_vio(vdo_waitq_dequeue_waiter(&block->entry_waiters)); in add_queued_recovery_entries()
1333 struct tree_lock *lock = &data_vio->tree_lock; in add_queued_recovery_entries()
1337 if (block->sector->entry_count == RECOVERY_JOURNAL_ENTRIES_PER_SECTOR) in add_queued_recovery_entries()
1339 (char *) block->sector + VDO_SECTOR_SIZE); in add_queued_recovery_entries()
1342 packed_entry = &block->sector->entries[block->sector->entry_count++]; in add_queued_recovery_entries()
1345 .pbn = data_vio->increment_updater.zpbn.pbn, in add_queued_recovery_entries()
1346 .state = data_vio->increment_updater.zpbn.state, in add_queued_recovery_entries()
1349 .pbn = data_vio->decrement_updater.zpbn.pbn, in add_queued_recovery_entries()
1350 .state = data_vio->decrement_updater.zpbn.state, in add_queued_recovery_entries()
1352 .operation = data_vio->increment_updater.operation, in add_queued_recovery_entries()
1353 .slot = lock->tree_slots[lock->height].block_map_slot, in add_queued_recovery_entries()
1356 data_vio->recovery_sequence_number = block->sequence_number; in add_queued_recovery_entries()
1359 vdo_waitq_enqueue_waiter(&block->commit_waiters, &data_vio->waiter); in add_queued_recovery_entries()
1364 * write_block() - Issue a block for writing.
1372 struct recovery_journal *journal = block->journal; in write_block()
1375 if (block->committing || !vdo_waitq_has_waiters(&block->entry_waiters) || in write_block()
1379 block->entries_in_commit = vdo_waitq_num_waiters(&block->entry_waiters); in write_block()
1382 journal->pending_write_count += 1; in write_block()
1383 journal->events.blocks.written += 1; in write_block()
1384 journal->events.entries.written += block->entries_in_commit; in write_block()
1386 header->block_map_head = __cpu_to_le64(journal->block_map_head); in write_block()
1387 header->slab_journal_head = __cpu_to_le64(journal->slab_journal_head); in write_block()
1388 header->entry_count = __cpu_to_le16(block->entry_count); in write_block()
1390 block->committing = true; in write_block()
1397 vdo_submit_metadata_vio(&block->vio, journal->origin + block->block_number, in write_block()
1404 * write_blocks() - Attempt to commit blocks, according to write policy.
1405 * @journal: The recovery journal.
1419 if (journal->pending_write_count > 0) in write_blocks()
1423 vdo_waitq_notify_all_waiters(&journal->pending_writes, write_block, NULL); in write_blocks()
1429 if ((journal->pending_write_count == 0) && (journal->active_block != NULL)) in write_blocks()
1430 write_block(&journal->active_block->write_waiter, NULL); in write_blocks()
1434 * vdo_add_recovery_journal_entry() - Add an entry to a recovery journal.
1443 * to the on-disk journal.
1449 if (!vdo_is_state_normal(&journal->state)) { in vdo_add_recovery_journal_entry()
1459 VDO_ASSERT_LOG_ONLY(data_vio->recovery_sequence_number == 0, in vdo_add_recovery_journal_entry()
1462 vdo_advance_journal_point(&journal->append_point, journal->entries_per_block); in vdo_add_recovery_journal_entry()
1463 vdo_waitq_enqueue_waiter(&journal->entry_waiters, &data_vio->waiter); in vdo_add_recovery_journal_entry()
1468 * is_lock_locked() - Check whether a lock is locked for a zone type.
1469 * @journal: The recovery journal.
1473 * If the recovery journal has a lock on the lock number, both logical and physical zones are
1495 * reap_recovery_journal() - Conduct a sweep on a recovery journal to reclaim unreferenced blocks.
1496 * @journal: The recovery journal.
1500 if (journal->reaping) { in reap_recovery_journal()
1508 if (vdo_is_state_quiescent(&journal->state)) { in reap_recovery_journal()
1517 while ((journal->block_map_reap_head < journal->last_write_acknowledged) && in reap_recovery_journal()
1518 !is_lock_locked(journal, journal->block_map_head_block_number, in reap_recovery_journal()
1520 journal->block_map_reap_head++; in reap_recovery_journal()
1521 if (++journal->block_map_head_block_number == journal->size) in reap_recovery_journal()
1522 journal->block_map_head_block_number = 0; in reap_recovery_journal()
1525 while ((journal->slab_journal_reap_head < journal->last_write_acknowledged) && in reap_recovery_journal()
1526 !is_lock_locked(journal, journal->slab_journal_head_block_number, in reap_recovery_journal()
1528 journal->slab_journal_reap_head++; in reap_recovery_journal()
1529 if (++journal->slab_journal_head_block_number == journal->size) in reap_recovery_journal()
1530 journal->slab_journal_head_block_number = 0; in reap_recovery_journal()
1533 if ((journal->block_map_reap_head == journal->block_map_head) && in reap_recovery_journal()
1534 (journal->slab_journal_reap_head == journal->slab_journal_head)) { in reap_recovery_journal()
1544 journal->reaping = true; in reap_recovery_journal()
1545 vdo_submit_flush_vio(journal->flush_vio, flush_endio, handle_flush_error); in reap_recovery_journal()
1549 * vdo_acquire_recovery_journal_block_reference() - Acquire a reference to a recovery journal block
1551 * @journal: The recovery journal.
1590 * vdo_release_journal_entry_lock() - Release a single per-entry reference count for a recovery
1592 * @journal: The recovery journal.
1615 * initiate_drain() - Initiate a drain.
1625 * vdo_drain_recovery_journal() - Drain recovery journal I/O.
1637 vdo_start_draining(&journal->state, operation, parent, initiate_drain); in vdo_drain_recovery_journal()
1641 * resume_lock_counter() - Re-allow notifications from a suspended lock counter.
1655 prior_state = atomic_cmpxchg(&counter->state, LOCK_COUNTER_STATE_SUSPENDED, in resume_lock_counter()
1664 * vdo_resume_recovery_journal() - Resume a recovery journal which has been drained.
1674 saved = vdo_is_state_saved(&journal->state); in vdo_resume_recovery_journal()
1675 vdo_set_completion_result(parent, vdo_resume_if_quiescent(&journal->state)); in vdo_resume_recovery_journal()
1684 if (resume_lock_counter(&journal->lock_counter)) { in vdo_resume_recovery_journal()
1693 * vdo_get_recovery_journal_logical_blocks_used() - Get the number of logical blocks in use by the
1701 return journal->logical_blocks_used; in vdo_get_recovery_journal_logical_blocks_used()
1705 * vdo_get_recovery_journal_statistics() - Get the current statistics from the recovery journal.
1706 * @journal: The recovery journal to query.
1713 return journal->events; in vdo_get_recovery_journal_statistics()
1717 * dump_recovery_block() - Dump the contents of the recovery block to the log.
1723 (unsigned long long) block->sequence_number, block->entry_count, in dump_recovery_block()
1724 (block->committing ? "committing" : "waiting"), in dump_recovery_block()
1725 vdo_waitq_num_waiters(&block->entry_waiters), in dump_recovery_block()
1726 vdo_waitq_num_waiters(&block->commit_waiters)); in dump_recovery_block()
1730 * vdo_dump_recovery_journal_statistics() - Dump some current statistics and other debug info from
1731 * the recovery journal.
1732 * @journal: The recovery journal to dump.
1739 vdo_log_info("Recovery Journal"); in vdo_dump_recovery_journal_statistics()
1741 (unsigned long long) journal->block_map_head, in vdo_dump_recovery_journal_statistics()
1742 (unsigned long long) journal->slab_journal_head, in vdo_dump_recovery_journal_statistics()
1743 (unsigned long long) journal->last_write_acknowledged, in vdo_dump_recovery_journal_statistics()
1744 (unsigned long long) journal->tail, in vdo_dump_recovery_journal_statistics()
1745 (unsigned long long) journal->block_map_reap_head, in vdo_dump_recovery_journal_statistics()
1746 (unsigned long long) journal->slab_journal_reap_head, in vdo_dump_recovery_journal_statistics()
1749 vdo_waitq_num_waiters(&journal->entry_waiters)); in vdo_dump_recovery_journal_statistics()
1760 list_for_each_entry(block, &journal->active_tail_blocks, list_node) in vdo_dump_recovery_journal_statistics()