18 #include "completion.h"
101 	struct vdo_completion *completion;  member
165 	struct vdo_page_completion *completion;  in page_completion_from_waiter()  local
170 	completion = container_of(waiter, struct vdo_page_completion, waiter);  in page_completion_from_waiter()
171 	vdo_assert_completion_type(&completion->completion, VDO_PAGE_COMPLETION);  in page_completion_from_waiter()
172 	return completion;  in page_completion_from_waiter()
199 		info->vio->completion.callback_thread_id = cache->zone->thread_id;  in initialize_info()
476  * complete_with_page() - Helper to complete the VDO Page Completion request successfully.
478  * @vdo_page_comp: The VDO page completion to complete.
491 		vdo_fail_completion(&vdo_page_comp->completion, VDO_BAD_PAGE);  in complete_with_page()
497 	vdo_finish_completion(&vdo_page_comp->completion);  in complete_with_page()
501  * complete_waiter_with_error() - Complete a page completion with an error code.
502  * @waiter: The page completion, as a waiter.
511 	vdo_fail_completion(&page_completion_from_waiter(waiter)->completion, *result);  in complete_waiter_with_error()
515  * complete_waiter_with_page() - Complete a page completion with a page.
516  * @waiter: The page completion, as a waiter.
529  * Upon completion the waitq will be empty.
542 	 * Increment the busy count once for each pending completion so that this page does not  in distribute_page_over_waitq()
584  * validate_completed_page() - Check that a page completion which is being freed to the cache
590 static int __must_check validate_completed_page(struct vdo_page_completion *completion,  in validate_completed_page()  argument
595 	result = VDO_ASSERT(completion->ready, "VDO Page completion not ready");  in validate_completed_page()
599 	result = VDO_ASSERT(completion->info != NULL,  in validate_completed_page()
600 			    "VDO Page Completion must be complete");  in validate_completed_page()
604 	result = VDO_ASSERT(completion->info->pbn == completion->pbn,  in validate_completed_page()
605 			    "VDO Page Completion pbn must be consistent");  in validate_completed_page()
609 	result = VDO_ASSERT(is_valid(completion->info),  in validate_completed_page()
610 			    "VDO Page Completion page must be valid");  in validate_completed_page()
615 		result = VDO_ASSERT(completion->writable,  in validate_completed_page()
616 				    "VDO Page Completion must be writable");  in validate_completed_page()
651 validate_completed_page_or_enter_read_only_mode(struct vdo_page_completion *completion,  in validate_completed_page_or_enter_read_only_mode()  argument
654 	int result = validate_completed_page(completion, writable);  in validate_completed_page_or_enter_read_only_mode()
659 	enter_zone_read_only_mode(completion->info->cache->zone, result);  in validate_completed_page_or_enter_read_only_mode()
665  * @completion: The page read vio.
667 static void handle_load_error(struct vdo_completion *completion)  in handle_load_error()  argument
669 	int result = completion->result;  in handle_load_error()
670 	struct page_info *info = completion->parent;  in handle_load_error()
674 	vio_record_metadata_io_error(as_vio(completion));  in handle_load_error()
691  * @completion: The vio which has loaded the page. Its parent is the page_info.
693 static void page_is_loaded(struct vdo_completion *completion)  in page_is_loaded()  argument
695 	struct page_info *info = completion->parent;  in page_is_loaded()
712 		vdo_continue_completion(completion, result);  in page_is_loaded()
733  * @completion: The page load completion.
735 static void handle_rebuild_read_error(struct vdo_completion *completion)  in handle_rebuild_read_error()  argument
737 	struct page_info *info = completion->parent;  in handle_rebuild_read_error()
746 	vio_record_metadata_io_error(as_vio(completion));  in handle_rebuild_read_error()
749 	vdo_reset_completion(completion);  in handle_rebuild_read_error()
750 	page_is_loaded(completion);  in handle_rebuild_read_error()
756 	struct page_info *info = vio->completion.parent;  in load_cache_page_endio()
792 static void write_pages(struct vdo_completion *completion);
795 static void handle_flush_error(struct vdo_completion *completion)  in handle_flush_error()  argument
797 	struct page_info *info = completion->parent;  in handle_flush_error()
799 	vio_record_metadata_io_error(as_vio(completion));  in handle_flush_error()
800 	set_persistent_error(info->cache, "flush failed", completion->result);  in handle_flush_error()
801 	write_pages(completion);  in handle_flush_error()
807 	struct page_info *info = vio->completion.parent;  in flush_endio()
873  * Return: true if the page completion is for the desired page number.
883  * allocate_free_page() - Allocate a free page to the first completion in the waiting queue, and
935  * If the selected page is not dirty, immediately allocates the page to the oldest completion
961  * discard_page_for_completion() - Helper used to trigger a discard so that the completion can get
1003  * @completion: The page write vio.
1005 static void handle_page_write_error(struct vdo_completion *completion)  in handle_page_write_error()  argument
1007 	int result = completion->result;  in handle_page_write_error()
1008 	struct page_info *info = completion->parent;  in handle_page_write_error()
1011 	vio_record_metadata_io_error(as_vio(completion));  in handle_page_write_error()
1030 static void page_is_written_out(struct vdo_completion *completion);
1035 	struct page_info *info = vio->completion.parent;  in write_cache_page_endio()
1042  * @completion: The vio which wrote the page. Its parent is a page_info.
1044 static void page_is_written_out(struct vdo_completion *completion)  in page_is_written_out()  argument
1048 	struct page_info *info = completion->parent;  in page_is_written_out()
1113 			struct vdo_completion *completion = &info->vio->completion;  in write_pages()  local
1115 			vdo_reset_completion(completion);  in write_pages()
1116 			completion->callback = page_is_written_out;  in write_pages()
1117 			completion->error_handler = handle_page_write_error;  in write_pages()
1118 			vdo_fail_completion(completion, VDO_READ_ONLY);  in write_pages()
1136  * vdo_release_page_completion() - Release a VDO Page Completion.
1138  * The page referenced by this completion (if any) will no longer be held busy by this completion.
1142 void vdo_release_page_completion(struct vdo_completion *completion)  in vdo_release_page_completion()  argument
1145 	struct vdo_page_completion *page_completion = as_vdo_page_completion(completion);  in vdo_release_page_completion()
1148 	if (completion->result == VDO_SUCCESS) {  in vdo_release_page_completion()
1179  *                              Completion.
1195  * vdo_get_page() - Initialize a page completion and get a block map page.
1206  * by the completion to be loaded from disk. When the callback is invoked, the page will be
1216 	struct vdo_completion *completion = &page_completion->completion;  in vdo_get_page()  local
1221 			    "New page completion was not already on a wait queue");  in vdo_get_page()
1229 	vdo_initialize_completion(completion, cache->vdo, VDO_PAGE_COMPLETION);  in vdo_get_page()
1230 	vdo_prepare_completion(completion, callback, error_handler,  in vdo_get_page()
1232 	completion->requeue = requeue;  in vdo_get_page()
1235 		vdo_fail_completion(completion, VDO_READ_ONLY);  in vdo_get_page()
1286  * @completion: The vdo_page_completion containing the page.
1288 void vdo_request_page_write(struct vdo_completion *completion)  in vdo_request_page_write()  argument
1291 	struct vdo_page_completion *vdo_page_comp = as_vdo_page_completion(completion);  in vdo_request_page_write()
1302  * vdo_get_cached_page() - Get the block map page from a page completion.
1303  * @completion: A vdo page completion whose callback has been called.
1308 int vdo_get_cached_page(struct vdo_completion *completion,  in vdo_get_cached_page()  argument
1314 	vpc = as_vdo_page_completion(completion);  in vdo_get_cached_page()
1554 static void finish_page_write(struct vdo_completion *completion)  in finish_page_write()  argument
1557 	struct vio *vio = as_vio(completion);  in finish_page_write()
1559 	struct tree_page *page = completion->parent;  in finish_page_write()
1600 static void handle_write_error(struct vdo_completion *completion)  in handle_write_error()  argument
1602 	int result = completion->result;  in handle_write_error()
1603 	struct vio *vio = as_vio(completion);  in handle_write_error()
1614 static void write_initialized_page(struct vdo_completion *completion)  in write_initialized_page()  argument
1616 	struct vio *vio = as_vio(completion);  in write_initialized_page()
1619 	struct tree_page *tree_page = completion->parent;  in write_initialized_page()
1651 	struct vdo_completion *completion = &vio->vio.completion;  in write_page()  local
1666 	completion->parent = tree_page;  in write_page()
1668 	completion->callback_thread_id = zone->thread_id;  in write_page()
1684 		write_initialized_page(completion);  in write_page()
1720 	data_vio->vio.completion.error_handler = handle_data_vio_error;  in finish_lookup()
1822 static void finish_block_map_page_load(struct vdo_completion *completion)  in finish_block_map_page_load()  argument
1828 	struct vio *vio = as_vio(completion);  in finish_block_map_page_load()
1830 	struct data_vio *data_vio = completion->parent;  in finish_block_map_page_load()
1850 static void handle_io_error(struct vdo_completion *completion)  in handle_io_error()  argument
1852 	int result = completion->result;  in handle_io_error()
1853 	struct vio *vio = as_vio(completion);  in handle_io_error()
1855 	struct data_vio *data_vio = completion->parent;  in handle_io_error()
1866 	struct data_vio *data_vio = vio->completion.parent;  in load_page_endio()
1879 	pooled->vio.completion.parent = data_vio;  in load_page()
1938 static void allocation_failure(struct vdo_completion *completion)  in allocation_failure()  argument
1940 	struct data_vio *data_vio = as_data_vio(completion);  in allocation_failure()
1942 	if (vdo_requeue_completion_if_needed(completion,  in allocation_failure()
1946 	abort_lookup(data_vio, completion->result, "allocation");  in allocation_failure()
2067 static void finish_block_map_allocation(struct vdo_completion *completion)  in finish_block_map_allocation()  argument
2073 	struct data_vio *data_vio = as_data_vio(completion);  in finish_block_map_allocation()
2128 static void release_block_map_write_lock(struct vdo_completion *completion)  in release_block_map_write_lock()  argument
2130 	struct data_vio *data_vio = as_data_vio(completion);  in release_block_map_write_lock()
2143 static void set_block_map_page_reference_count(struct vdo_completion *completion)  in set_block_map_page_reference_count()  argument
2145 	struct data_vio *data_vio = as_data_vio(completion);  in set_block_map_page_reference_count()
2149 	completion->callback = release_block_map_write_lock;  in set_block_map_page_reference_count()
2150 	vdo_modify_reference_count(completion, &data_vio->increment_updater);  in set_block_map_page_reference_count()
2153 static void journal_block_map_allocation(struct vdo_completion *completion)  in journal_block_map_allocation()  argument
2155 	struct data_vio *data_vio = as_data_vio(completion);  in journal_block_map_allocation()
2161 	vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio);  in journal_block_map_allocation()
2164 static void allocate_block(struct vdo_completion *completion)  in allocate_block()  argument
2166 	struct data_vio *data_vio = as_data_vio(completion);  in allocate_block()
2502 	struct vdo_completion *completion = cursors->completion;  in finish_cursor()  local
2510 	vdo_finish_completion(completion);  in finish_cursor()
2517  * @completion: The VIO doing a read or write.
2519 static void continue_traversal(struct vdo_completion *completion)  in continue_traversal()  argument
2521 	vio_record_metadata_io_error(as_vio(completion));  in continue_traversal()
2522 	traverse(completion->parent);  in continue_traversal()
2527  * @completion: The VIO doing the read.
2529 static void finish_traversal_load(struct vdo_completion *completion)  in finish_traversal_load()  argument
2531 	struct cursor *cursor = completion->parent;  in finish_traversal_load()
2547 	struct cursor *cursor = vio->completion.parent;  in traversal_endio()
2596 									    cursor->parent->completion);  in traverse()
2635 	pooled->vio.completion.parent = cursor;  in launch_cursor()
2636 	pooled->vio.completion.callback_thread_id = cursor->parent->zone->thread_id;  in launch_cursor()
2675  * @completion: The completion to notify on each traversed PBN, and when traversal completes.
2678 			 struct vdo_completion *completion)  in vdo_traverse_forest()  argument
2687 		vdo_fail_completion(completion, result);  in vdo_traverse_forest()
2694 	cursors->completion = completion;  in vdo_traverse_forest()
3040 static void grow_forest(void *context, struct vdo_completion *completion)  in grow_forest()  argument
3043 	vdo_finish_completion(completion);  in grow_forest()
3064 /* Release the page completion and then continue the requester. */
3065 static inline void finish_processing_page(struct vdo_completion *completion, int result)  in finish_processing_page()  argument
3067 	struct vdo_completion *parent = completion->parent;  in finish_processing_page()
3069 	vdo_release_page_completion(completion);  in finish_processing_page()
3073 static void handle_page_error(struct vdo_completion *completion)  in handle_page_error()  argument
3075 	finish_processing_page(completion, completion->result);  in handle_page_error()
3091 		     modifiable, &data_vio->vio.completion,  in fetch_mapping_page()
3163 static void get_mapping_from_fetched_page(struct vdo_completion *completion)  in get_mapping_from_fetched_page()  argument
3166 	struct vdo_page_completion *vpc = as_vdo_page_completion(completion);  in get_mapping_from_fetched_page()
3169 	struct data_vio *data_vio = as_data_vio(completion->parent);  in get_mapping_from_fetched_page()
3172 	if (completion->result != VDO_SUCCESS) {  in get_mapping_from_fetched_page()
3173 		finish_processing_page(completion, completion->result);  in get_mapping_from_fetched_page()
3179 		finish_processing_page(completion, result);  in get_mapping_from_fetched_page()
3188 	finish_processing_page(completion, result);  in get_mapping_from_fetched_page()
3232 static void put_mapping_in_fetched_page(struct vdo_completion *completion)  in put_mapping_in_fetched_page()  argument
3234 	struct data_vio *data_vio = as_data_vio(completion->parent);  in put_mapping_in_fetched_page()
3240 	if (completion->result != VDO_SUCCESS) {  in put_mapping_in_fetched_page()
3241 		finish_processing_page(completion, completion->result);  in put_mapping_in_fetched_page()
3245 	vpc = as_vdo_page_completion(completion);  in put_mapping_in_fetched_page()
3248 		finish_processing_page(completion, result);  in put_mapping_in_fetched_page()
3260 	finish_processing_page(completion, VDO_SUCCESS);  in put_mapping_in_fetched_page()