Lines Matching refs:repair
172 static struct numbered_block_mapping *sort_next_heap_element(struct repair_completion *repair) in sort_next_heap_element() argument
174 struct replay_heap *heap = &repair->replay_heap; in sort_next_heap_element()
184 last = &repair->entries[--heap->nr]; in sort_next_heap_element()
203 static void prepare_repair_completion(struct repair_completion *repair, in prepare_repair_completion() argument
206 struct vdo_completion *completion = &repair->completion; in prepare_repair_completion()
218 static void launch_repair_completion(struct repair_completion *repair, in launch_repair_completion() argument
221 prepare_repair_completion(repair, callback, zone_type); in launch_repair_completion()
222 vdo_launch_completion(&repair->completion); in launch_repair_completion()
225 static void uninitialize_vios(struct repair_completion *repair) in uninitialize_vios() argument
227 while (repair->vio_count > 0) in uninitialize_vios()
228 free_vio_components(&repair->vios[--repair->vio_count]); in uninitialize_vios()
230 vdo_free(vdo_forget(repair->vios)); in uninitialize_vios()
233 static void free_repair_completion(struct repair_completion *repair) in free_repair_completion() argument
235 if (repair == NULL) in free_repair_completion()
242 repair->completion.vdo->block_map->zones[0].page_cache.rebuilding = false; in free_repair_completion()
244 uninitialize_vios(repair); in free_repair_completion()
245 vdo_free(vdo_forget(repair->journal_data)); in free_repair_completion()
246 vdo_free(vdo_forget(repair->entries)); in free_repair_completion()
247 vdo_free(repair); in free_repair_completion()
254 struct repair_completion *repair = as_repair_completion(completion); in finish_repair() local
263 repair->highest_tail, in finish_repair()
264 repair->logical_blocks_used, in finish_repair()
265 repair->block_map_data_blocks); in finish_repair()
266 free_repair_completion(vdo_forget(repair)); in finish_repair()
292 struct repair_completion *repair = as_repair_completion(completion); in abort_repair() local
299 free_repair_completion(vdo_forget(repair)); in abort_repair()
310 static bool __must_check abort_on_error(int result, struct repair_completion *repair) in abort_on_error() argument
315 vdo_fail_completion(&repair->completion, result); in abort_on_error()
327 struct repair_completion *repair = as_repair_completion(completion); in drain_slab_depot() local
332 prepare_repair_completion(repair, finish_repair, VDO_ZONE_TYPE_ADMIN); in drain_slab_depot()
338 repair->entries_added_to_slab_journals); in drain_slab_depot()
362 static bool fetch_page(struct repair_completion *repair,
371 struct repair_completion *repair = completion->parent; in handle_page_load_error() local
373 repair->outstanding--; in handle_page_load_error()
374 vdo_set_completion_result(&repair->completion, completion->result); in handle_page_load_error()
376 fetch_page(repair, completion); in handle_page_load_error()
467 static void rebuild_reference_counts_from_page(struct repair_completion *repair, in rebuild_reference_counts_from_page() argument
476 vdo_set_completion_result(&repair->completion, result); in rebuild_reference_counts_from_page()
484 if (vdo_get_block_map_page_pbn(page) == repair->last_slot.pbn) { in rebuild_reference_counts_from_page()
485 last_slot = repair->last_slot.slot; in rebuild_reference_counts_from_page()
494 repair->logical_blocks_used++; in rebuild_reference_counts_from_page()
506 struct repair_completion *repair = completion->parent; in page_loaded() local
508 repair->outstanding--; in page_loaded()
509 rebuild_reference_counts_from_page(repair, completion); in page_loaded()
513 fetch_page(repair, completion); in page_loaded()
516 static physical_block_number_t get_pbn_to_fetch(struct repair_completion *repair, in get_pbn_to_fetch() argument
521 if (repair->completion.result != VDO_SUCCESS) in get_pbn_to_fetch()
524 while ((pbn == VDO_ZERO_BLOCK) && (repair->page_to_fetch < repair->leaf_pages)) in get_pbn_to_fetch()
525 pbn = vdo_find_block_map_page_pbn(block_map, repair->page_to_fetch++); in get_pbn_to_fetch()
527 if (vdo_is_physical_data_block(repair->completion.vdo->depot, pbn)) in get_pbn_to_fetch()
530 vdo_set_completion_result(&repair->completion, VDO_BAD_MAPPING); in get_pbn_to_fetch()
541 static bool fetch_page(struct repair_completion *repair, in fetch_page() argument
545 struct block_map *block_map = repair->completion.vdo->block_map; in fetch_page()
546 physical_block_number_t pbn = get_pbn_to_fetch(repair, block_map); in fetch_page()
549 repair->outstanding++; in fetch_page()
554 vdo_get_page(page_completion, &block_map->zones[0], pbn, true, repair, in fetch_page()
558 if (repair->outstanding > 0) in fetch_page()
561 launch_repair_completion(repair, flush_block_map_updates, VDO_ZONE_TYPE_ADMIN); in fetch_page()
576 struct repair_completion *repair = as_repair_completion(completion); in rebuild_from_leaves() local
579 repair->logical_blocks_used = 0; in rebuild_from_leaves()
585 repair->leaf_pages = vdo_compute_block_map_page_count(map->entry_count); in rebuild_from_leaves()
586 repair->last_slot = (struct block_map_slot) { in rebuild_from_leaves()
588 .pbn = vdo_find_block_map_page_pbn(map, repair->leaf_pages - 1), in rebuild_from_leaves()
590 if (repair->last_slot.slot == 0) in rebuild_from_leaves()
591 repair->last_slot.slot = VDO_BLOCK_MAP_ENTRIES_PER_PAGE; in rebuild_from_leaves()
593 for (i = 0; i < repair->page_count; i++) { in rebuild_from_leaves()
594 if (fetch_page(repair, &repair->page_completions[i].completion)) { in rebuild_from_leaves()
615 struct repair_completion *repair = as_repair_completion(completion); in process_entry() local
633 repair->block_map_data_blocks++; in process_entry()
639 struct repair_completion *repair = as_repair_completion(completion); in rebuild_reference_counts() local
644 if (abort_on_error(vdo_allocate_reference_counters(vdo->depot), repair)) in rebuild_reference_counts()
651 if (abort_on_error(vdo_invalidate_page_cache(cache), repair)) in rebuild_reference_counts()
654 prepare_repair_completion(repair, rebuild_from_leaves, VDO_ZONE_TYPE_LOGICAL); in rebuild_reference_counts()
678 static void advance_points(struct repair_completion *repair, in advance_points() argument
681 if (!repair->next_recovery_point.increment_applied) { in advance_points()
682 repair->next_recovery_point.increment_applied = true; in advance_points()
686 increment_recovery_point(&repair->next_recovery_point); in advance_points()
687 vdo_advance_journal_point(&repair->next_journal_point, entries_per_block); in advance_points()
688 repair->next_recovery_point.increment_applied = false; in advance_points()
733 static struct recovery_journal_entry get_entry(const struct repair_completion *repair, in get_entry() argument
738 sector = get_sector(repair->completion.vdo->recovery_journal, in get_entry()
739 repair->journal_data, point->sequence_number, in get_entry()
796 struct repair_completion *repair = completion->parent; in add_slab_journal_entries() local
804 completion->callback_thread_id, repair); in add_slab_journal_entries()
805 for (recovery_point = &repair->next_recovery_point; in add_slab_journal_entries()
806 before_recovery_point(recovery_point, &repair->tail_recovery_point); in add_slab_journal_entries()
807 advance_points(repair, journal->entries_per_block)) { in add_slab_journal_entries()
811 struct recovery_journal_entry entry = get_entry(repair, recovery_point); in add_slab_journal_entries()
812 bool increment = !repair->next_recovery_point.increment_applied; in add_slab_journal_entries()
835 &repair->next_journal_point, in add_slab_journal_entries()
839 repair->entries_added_to_slab_journals++; in add_slab_journal_entries()
854 struct repair_completion *repair = context; in vdo_replay_into_slab_journals() local
858 if (repair->entry_count == 0) { in vdo_replay_into_slab_journals()
860 repair->logical_blocks_used = vdo->recovery_journal->logical_blocks_used; in vdo_replay_into_slab_journals()
861 repair->block_map_data_blocks = vdo->recovery_journal->block_map_data_blocks; in vdo_replay_into_slab_journals()
866 repair->next_recovery_point = (struct recovery_point) { in vdo_replay_into_slab_journals()
867 .sequence_number = repair->slab_journal_head, in vdo_replay_into_slab_journals()
872 repair->next_journal_point = (struct journal_point) { in vdo_replay_into_slab_journals()
873 .sequence_number = repair->slab_journal_head, in vdo_replay_into_slab_journals()
879 completion->parent = repair; in vdo_replay_into_slab_journals()
885 struct repair_completion *repair = as_repair_completion(completion); in load_slab_depot() local
891 prepare_repair_completion(repair, rebuild_reference_counts, in load_slab_depot()
895 prepare_repair_completion(repair, drain_slab_depot, VDO_ZONE_TYPE_ADMIN); in load_slab_depot()
899 vdo_load_slab_depot(completion->vdo->depot, operation, completion, repair); in load_slab_depot()
904 struct repair_completion *repair = as_repair_completion(completion); in flush_block_map() local
910 prepare_repair_completion(repair, load_slab_depot, VDO_ZONE_TYPE_ADMIN); in flush_block_map()
917 static bool finish_if_done(struct repair_completion *repair) in finish_if_done() argument
920 if (repair->launching || (repair->outstanding > 0)) in finish_if_done()
923 if (repair->completion.result != VDO_SUCCESS) { in finish_if_done()
926 for (i = 0; i < repair->page_count; i++) { in finish_if_done()
928 &repair->page_completions[i]; in finish_if_done()
934 vdo_launch_completion(&repair->completion); in finish_if_done()
938 if (repair->current_entry >= repair->entries) in finish_if_done()
941 launch_repair_completion(repair, flush_block_map, VDO_ZONE_TYPE_ADMIN); in finish_if_done()
945 static void abort_block_map_recovery(struct repair_completion *repair, int result) in abort_block_map_recovery() argument
947 vdo_set_completion_result(&repair->completion, result); in abort_block_map_recovery()
948 finish_if_done(repair); in abort_block_map_recovery()
962 find_entry_starting_next_page(struct repair_completion *repair, in find_entry_starting_next_page() argument
968 if (current_entry < repair->entries) in find_entry_starting_next_page()
974 while ((current_entry >= repair->entries) && in find_entry_starting_next_page()
978 sort_next_heap_element(repair); in find_entry_starting_next_page()
1005 static void recover_ready_pages(struct repair_completion *repair,
1010 struct repair_completion *repair = as_repair_completion(completion->parent); in block_map_page_loaded() local
1012 repair->outstanding--; in block_map_page_loaded()
1013 if (!repair->launching) in block_map_page_loaded()
1014 recover_ready_pages(repair, completion); in block_map_page_loaded()
1019 struct repair_completion *repair = as_repair_completion(completion->parent); in handle_block_map_page_load_error() local
1021 repair->outstanding--; in handle_block_map_page_load_error()
1022 abort_block_map_recovery(repair, completion->result); in handle_block_map_page_load_error()
1025 static void fetch_block_map_page(struct repair_completion *repair, in fetch_block_map_page() argument
1030 if (repair->current_unfetched_entry < repair->entries) in fetch_block_map_page()
1035 pbn = repair->current_unfetched_entry->block_map_slot.pbn; in fetch_block_map_page()
1036 repair->current_unfetched_entry = in fetch_block_map_page()
1037 find_entry_starting_next_page(repair, repair->current_unfetched_entry, in fetch_block_map_page()
1039 repair->outstanding++; in fetch_block_map_page()
1041 &repair->completion.vdo->block_map->zones[0], pbn, true, in fetch_block_map_page()
1042 &repair->completion, block_map_page_loaded, in fetch_block_map_page()
1046 static struct vdo_page_completion *get_next_page_completion(struct repair_completion *repair, in get_next_page_completion() argument
1050 if (completion == (&repair->page_completions[repair->page_count])) in get_next_page_completion()
1051 completion = &repair->page_completions[0]; in get_next_page_completion()
1055 static void recover_ready_pages(struct repair_completion *repair, in recover_ready_pages() argument
1060 if (finish_if_done(repair)) in recover_ready_pages()
1063 if (repair->pbn != page_completion->pbn) in recover_ready_pages()
1073 abort_block_map_recovery(repair, result); in recover_ready_pages()
1078 find_entry_starting_next_page(repair, repair->current_entry, in recover_ready_pages()
1080 apply_journal_entries_to_page(page, repair->current_entry, in recover_ready_pages()
1082 repair->current_entry = start_of_next_page; in recover_ready_pages()
1086 if (finish_if_done(repair)) in recover_ready_pages()
1089 repair->pbn = repair->current_entry->block_map_slot.pbn; in recover_ready_pages()
1090 fetch_block_map_page(repair, completion); in recover_ready_pages()
1091 page_completion = get_next_page_completion(repair, page_completion); in recover_ready_pages()
1098 struct repair_completion *repair = as_repair_completion(completion); in recover_block_map() local
1109 if (repair->block_map_entry_count == 0) { in recover_block_map()
1111 vdo_free(vdo_forget(repair->journal_data)); in recover_block_map()
1112 launch_repair_completion(repair, load_slab_depot, VDO_ZONE_TYPE_ADMIN); in recover_block_map()
1120 repair->replay_heap = (struct replay_heap) { in recover_block_map()
1121 .data = repair->entries, in recover_block_map()
1122 .nr = repair->block_map_entry_count, in recover_block_map()
1123 .size = repair->block_map_entry_count, in recover_block_map()
1125 min_heapify_all(&repair->replay_heap, &repair_min_heap, NULL); in recover_block_map()
1128 repair->block_map_entry_count); in recover_block_map()
1130 repair->current_entry = &repair->entries[repair->block_map_entry_count - 1]; in recover_block_map()
1131 first_sorted_entry = sort_next_heap_element(repair); in recover_block_map()
1132 VDO_ASSERT_LOG_ONLY(first_sorted_entry == repair->current_entry, in recover_block_map()
1136 repair->launching = true; in recover_block_map()
1137 repair->pbn = repair->current_entry->block_map_slot.pbn; in recover_block_map()
1138 repair->current_unfetched_entry = repair->current_entry; in recover_block_map()
1139 for (i = 0; i < repair->page_count; i++) { in recover_block_map()
1140 if (repair->current_unfetched_entry < repair->entries) in recover_block_map()
1143 fetch_block_map_page(repair, &repair->page_completions[i].completion); in recover_block_map()
1145 repair->launching = false; in recover_block_map()
1148 recover_ready_pages(repair, &repair->page_completions[0].completion); in recover_block_map()
1222 static bool find_recovery_journal_head_and_tail(struct repair_completion *repair) in find_recovery_journal_head_and_tail() argument
1224 struct recovery_journal *journal = repair->completion.vdo->recovery_journal; in find_recovery_journal_head_and_tail()
1233 repair->highest_tail = journal->tail; in find_recovery_journal_head_and_tail()
1236 get_recovery_journal_block_header(journal, repair->journal_data, i); in find_recovery_journal_head_and_tail()
1248 if (header.sequence_number >= repair->highest_tail) { in find_recovery_journal_head_and_tail()
1250 repair->highest_tail = header.sequence_number; in find_recovery_journal_head_and_tail()
1256 if (header.block_map_head > repair->block_map_head) in find_recovery_journal_head_and_tail()
1257 repair->block_map_head = header.block_map_head; in find_recovery_journal_head_and_tail()
1259 if (header.slab_journal_head > repair->slab_journal_head) in find_recovery_journal_head_and_tail()
1260 repair->slab_journal_head = header.slab_journal_head; in find_recovery_journal_head_and_tail()
1321 static void append_sector_entries(struct repair_completion *repair, char *entries, in append_sector_entries() argument
1326 struct vdo *vdo = repair->completion.vdo; in append_sector_entries()
1338 repair->entries[repair->block_map_entry_count] = in append_sector_entries()
1343 .number = repair->block_map_entry_count, in append_sector_entries()
1345 repair->block_map_entry_count++; in append_sector_entries()
1360 static void extract_entries_from_block(struct repair_completion *repair, in extract_entries_from_block() argument
1368 get_recovery_journal_block_header(journal, repair->journal_data, in extract_entries_from_block()
1380 get_sector(journal, repair->journal_data, sequence, i); in extract_entries_from_block()
1386 append_sector_entries(repair, (char *) sector->entries, format, in extract_entries_from_block()
1400 static int parse_journal_for_rebuild(struct repair_completion *repair) in parse_journal_for_rebuild() argument
1406 struct vdo *vdo = repair->completion.vdo; in parse_journal_for_rebuild()
1410 format = get_recovery_journal_block_header(journal, repair->journal_data, in parse_journal_for_rebuild()
1411 repair->highest_tail).metadata_type; in parse_journal_for_rebuild()
1419 count = ((repair->highest_tail - repair->block_map_head + 1) * entries_per_block); in parse_journal_for_rebuild()
1421 &repair->entries); in parse_journal_for_rebuild()
1425 for (i = repair->block_map_head; i <= repair->highest_tail; i++) in parse_journal_for_rebuild()
1426 extract_entries_from_block(repair, journal, i, format, entries_per_block); in parse_journal_for_rebuild()
1431 static int validate_heads(struct repair_completion *repair) in validate_heads() argument
1434 if ((repair->block_map_head <= repair->tail) && in validate_heads()
1435 (repair->slab_journal_head <= repair->tail)) in validate_heads()
1441 (unsigned long long) repair->block_map_head, in validate_heads()
1442 (unsigned long long) repair->slab_journal_head, in validate_heads()
1443 (unsigned long long) repair->tail); in validate_heads()
1453 static int extract_new_mappings(struct repair_completion *repair) in extract_new_mappings() argument
1456 struct vdo *vdo = repair->completion.vdo; in extract_new_mappings()
1458 .sequence_number = repair->block_map_head, in extract_new_mappings()
1467 result = vdo_allocate(repair->entry_count, struct numbered_block_mapping, in extract_new_mappings()
1468 __func__, &repair->entries); in extract_new_mappings()
1472 for (; before_recovery_point(&recovery_point, &repair->tail_recovery_point); in extract_new_mappings()
1474 struct recovery_journal_entry entry = get_entry(repair, &recovery_point); in extract_new_mappings()
1482 repair->entries[repair->block_map_entry_count] = in extract_new_mappings()
1487 .number = repair->block_map_entry_count, in extract_new_mappings()
1489 repair->block_map_entry_count++; in extract_new_mappings()
1492 result = VDO_ASSERT((repair->block_map_entry_count <= repair->entry_count), in extract_new_mappings()
1505 static noinline int compute_usages(struct repair_completion *repair) in compute_usages() argument
1512 .sequence_number = repair->tail, in compute_usages()
1517 struct vdo *vdo = repair->completion.vdo; in compute_usages()
1520 get_recovery_journal_block_header(journal, repair->journal_data, in compute_usages()
1521 repair->tail); in compute_usages()
1523 repair->logical_blocks_used = header.logical_blocks_used; in compute_usages()
1524 repair->block_map_data_blocks = header.block_map_data_blocks; in compute_usages()
1526 for (; before_recovery_point(&recovery_point, &repair->tail_recovery_point); in compute_usages()
1528 struct recovery_journal_entry entry = get_entry(repair, &recovery_point); in compute_usages()
1538 repair->block_map_data_blocks++; in compute_usages()
1543 repair->logical_blocks_used++; in compute_usages()
1546 repair->logical_blocks_used--; in compute_usages()
1552 static int parse_journal_for_recovery(struct repair_completion *repair) in parse_journal_for_recovery() argument
1557 struct recovery_journal *journal = repair->completion.vdo->recovery_journal; in parse_journal_for_recovery()
1561 head = min(repair->block_map_head, repair->slab_journal_head); in parse_journal_for_recovery()
1562 header = get_recovery_journal_block_header(journal, repair->journal_data, head); in parse_journal_for_recovery()
1564 for (i = head; i <= repair->highest_tail; i++) { in parse_journal_for_recovery()
1568 repair->tail = i; in parse_journal_for_recovery()
1569 repair->tail_recovery_point = (struct recovery_point) { in parse_journal_for_recovery()
1575 header = get_recovery_journal_block_header(journal, repair->journal_data, i); in parse_journal_for_recovery()
1583 vdo_enter_read_only_mode(repair->completion.vdo, VDO_CORRUPT_JOURNAL); in parse_journal_for_recovery()
1592 get_sector(journal, repair->journal_data, i, j); in parse_journal_for_recovery()
1603 repair->tail_recovery_point.sector_count++; in parse_journal_for_recovery()
1604 repair->tail_recovery_point.entry_count = sector_entries; in parse_journal_for_recovery()
1606 repair->entry_count += sector_entries; in parse_journal_for_recovery()
1621 return validate_heads(repair); in parse_journal_for_recovery()
1630 if (repair->tail_recovery_point.sector_count == 0) in parse_journal_for_recovery()
1631 repair->tail--; in parse_journal_for_recovery()
1633 result = validate_heads(repair); in parse_journal_for_recovery()
1638 (unsigned long long) repair->highest_tail, in parse_journal_for_recovery()
1639 (unsigned long long) repair->tail); in parse_journal_for_recovery()
1641 result = extract_new_mappings(repair); in parse_journal_for_recovery()
1645 return compute_usages(repair); in parse_journal_for_recovery()
1648 static int parse_journal(struct repair_completion *repair) in parse_journal() argument
1650 if (!find_recovery_journal_head_and_tail(repair)) in parse_journal()
1653 return (vdo_state_requires_read_only_rebuild(repair->completion.vdo->load_state) ? in parse_journal()
1654 parse_journal_for_rebuild(repair) : in parse_journal()
1655 parse_journal_for_recovery(repair)); in parse_journal()
1660 struct repair_completion *repair = completion->parent; in finish_journal_load() local
1662 if (++repair->vios_complete != repair->vio_count) in finish_journal_load()
1666 uninitialize_vios(repair); in finish_journal_load()
1667 prepare_repair_completion(repair, recover_block_map, VDO_ZONE_TYPE_LOGICAL); in finish_journal_load()
1668 vdo_continue_completion(&repair->completion, parse_journal(repair)); in finish_journal_load()
1673 struct repair_completion *repair = completion->parent; in handle_journal_load_error() local
1676 vdo_set_completion_result(&repair->completion, completion->result); in handle_journal_load_error()
1697 struct repair_completion *repair; in vdo_repair() local
1720 &repair); in vdo_repair()
1726 vdo_initialize_completion(&repair->completion, vdo, VDO_REPAIR_COMPLETION); in vdo_repair()
1727 repair->completion.error_handler = abort_repair; in vdo_repair()
1728 repair->completion.parent = parent; in vdo_repair()
1729 prepare_repair_completion(repair, finish_repair, VDO_ZONE_TYPE_ADMIN); in vdo_repair()
1730 repair->page_count = page_count; in vdo_repair()
1733 &repair->journal_data); in vdo_repair()
1734 if (abort_on_error(result, repair)) in vdo_repair()
1737 result = vdo_allocate(vio_count, struct vio, __func__, &repair->vios); in vdo_repair()
1738 if (abort_on_error(result, repair)) in vdo_repair()
1741 ptr = repair->journal_data; in vdo_repair()
1742 for (repair->vio_count = 0; repair->vio_count < vio_count; repair->vio_count++) { in vdo_repair()
1748 repair, blocks, ptr, in vdo_repair()
1749 &repair->vios[repair->vio_count]); in vdo_repair()
1750 if (abort_on_error(result, repair)) in vdo_repair()
1757 for (vio_count = 0; vio_count < repair->vio_count; in vdo_repair()
1759 vdo_submit_metadata_vio(&repair->vios[vio_count], pbn, read_journal_endio, in vdo_repair()