58  *   completion is required for further work to be done by the issuer.
101  * completion will be enqueued on a cpu queue. This obviates the need for the releasing threads to
105  * Whenever the pool's completion is run on a cpu thread, it calls process_release_callback() which
159 	/* Completion for scheduling releases */
160 	struct vdo_completion completion;  member
215 as_data_vio_pool(struct vdo_completion *completion)  in as_data_vio_pool()  argument
217 	vdo_assert_completion_type(completion, VDO_DATA_VIO_POOL_COMPLETION);  in as_data_vio_pool()
218 	return container_of(completion, struct data_vio_pool, completion);  in as_data_vio_pool()
275 	int error = vdo_status_to_errno(data_vio->vio.completion.result);  in acknowledge_data_vio()
414  * @completion: The data_vio for an external data request as a completion.
418 static void attempt_logical_block_lock(struct vdo_completion *completion)  in attempt_logical_block_lock()  argument
420 	struct data_vio *data_vio = as_data_vio(completion);  in attempt_logical_block_lock()
463 		complete_data_vio(completion);  in attempt_logical_block_lock()
487 	struct vdo_completion *completion = &data_vio->vio.completion;  in launch_data_vio()  local
505 	vdo_reset_completion(completion);  in launch_data_vio()
506 	completion->error_handler = handle_data_vio_error;  in launch_data_vio()
508 	vdo_enqueue_completion(completion, VDO_DEFAULT_Q_MAP_BIO_PRIORITY);  in launch_data_vio()
589 	launch_bio(limiter->pool->completion.vdo, data_vio, bio);  in assign_data_vio()
665 	pool->completion.requeue = true;  in schedule_releases()
666 	vdo_launch_completion_with_priority(&pool->completion,  in schedule_releases()
695  * @completion: The pool with data_vios to release.
697 static void process_release_callback(struct vdo_completion *completion)  in process_release_callback()  argument
699 	struct data_vio_pool *pool = as_data_vio_pool(completion);  in process_release_callback()
863 	vdo_initialize_completion(&pool->completion, vdo, VDO_DATA_VIO_POOL_COMPLETION);  in make_data_vio_pool()
864 	vdo_prepare_completion(&pool->completion, process_release_callback,  in make_data_vio_pool()
982 	launch_bio(pool->completion.vdo, data_vio, bio);  in vdo_launch_bio()
1007  * @completion: The completion to notify when the pool has drained.
1009 void drain_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion)  in drain_data_vio_pool()  argument
1011 	assert_on_vdo_cpu_thread(completion->vdo, __func__);  in drain_data_vio_pool()
1012 	vdo_start_draining(&pool->state, VDO_ADMIN_STATE_SUSPENDING, completion,  in drain_data_vio_pool()
1018  * @completion: The completion to notify when the pool has resumed.
1020 void resume_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion)  in resume_data_vio_pool()  argument
1022 	assert_on_vdo_cpu_thread(completion->vdo, __func__);  in resume_data_vio_pool()
1023 	vdo_continue_completion(completion, vdo_resume_if_quiescent(&pool->state));  in resume_data_vio_pool()
1157 static void release_allocated_lock(struct vdo_completion *completion)  in release_allocated_lock()  argument
1159 	struct data_vio *data_vio = as_data_vio(completion);  in release_allocated_lock()
1229 	next_lock_holder->vio.completion.requeue = true;  in transfer_lock()
1237 static void release_logical_lock(struct vdo_completion *completion)  in release_logical_lock()  argument
1239 	struct data_vio *data_vio = as_data_vio(completion);  in release_logical_lock()
1254 static void clean_hash_lock(struct vdo_completion *completion)  in clean_hash_lock()  argument
1256 	struct data_vio *data_vio = as_data_vio(completion);  in clean_hash_lock()
1259 	if (completion->result != VDO_SUCCESS) {  in clean_hash_lock()
1276 	struct vdo_completion *completion = &data_vio->vio.completion;  in finish_cleanup()  local
1285 	    (completion->result != VDO_SUCCESS)) {  in finish_cleanup()
1286 		struct data_vio_pool *pool = completion->vdo->data_vio_pool;  in finish_cleanup()
1288 		vdo_funnel_queue_put(pool->queue, &completion->work_queue_entry_link);  in finish_cleanup()
1297 	completion->requeue = true;  in finish_cleanup()
1327 		    (data_vio->vio.completion.result != VDO_READ_ONLY))  in perform_cleanup_stage()
1340 void complete_data_vio(struct vdo_completion *completion)  in complete_data_vio()  argument
1342 	struct data_vio *data_vio = as_data_vio(completion);  in complete_data_vio()
1344 	completion->error_handler = NULL;  in complete_data_vio()
1350 static void enter_read_only_mode(struct vdo_completion *completion)  in enter_read_only_mode()  argument
1352 	if (vdo_is_read_only(completion->vdo))  in enter_read_only_mode()
1355 	if (completion->result != VDO_READ_ONLY) {  in enter_read_only_mode()
1356 		struct data_vio *data_vio = as_data_vio(completion);  in enter_read_only_mode()
1358 		vdo_log_error_strerror(completion->result,  in enter_read_only_mode()
1367 	vdo_enter_read_only_mode(completion->vdo, completion->result);  in enter_read_only_mode()
1370 void handle_data_vio_error(struct vdo_completion *completion)  in handle_data_vio_error()  argument
1372 	struct data_vio *data_vio = as_data_vio(completion);  in handle_data_vio_error()
1374 	if ((completion->result == VDO_READ_ONLY) || (data_vio->user_bio == NULL))  in handle_data_vio_error()
1375 		enter_read_only_mode(completion);  in handle_data_vio_error()
1378 	complete_data_vio(completion);  in handle_data_vio_error()
1415 	data_vio->vio.completion.error_handler = error_handler;  in data_vio_allocate_data_block()
1470  * @completion: The data_vio which has just finished its read.
1474 static void modify_for_partial_write(struct vdo_completion *completion)  in modify_for_partial_write()  argument
1476 	struct data_vio *data_vio = as_data_vio(completion);  in modify_for_partial_write()
1496 static void complete_read(struct vdo_completion *completion)  in complete_read()  argument
1498 	struct data_vio *data_vio = as_data_vio(completion);  in complete_read()
1514 		modify_for_partial_write(completion);  in complete_read()
1522 	complete_data_vio(completion);  in complete_read()
1540 static void complete_zero_read(struct vdo_completion *completion)  in complete_zero_read()  argument
1542 	struct data_vio *data_vio = as_data_vio(completion);  in complete_zero_read()
1549 			modify_for_partial_write(completion);  in complete_zero_read()
1556 	complete_read(completion);  in complete_zero_read()
1564 static void read_block(struct vdo_completion *completion)  in read_block()  argument
1566 	struct data_vio *data_vio = as_data_vio(completion);  in read_block()
1567 	struct vio *vio = as_vio(completion);  in read_block()
1607 reference_count_update_completion_as_data_vio(struct vdo_completion *completion)  in reference_count_update_completion_as_data_vio()  argument
1609 	if (completion->type == VIO_COMPLETION)  in reference_count_update_completion_as_data_vio()
1610 		return as_data_vio(completion);  in reference_count_update_completion_as_data_vio()
1612 	return container_of(completion, struct data_vio, decrement_completion);  in reference_count_update_completion_as_data_vio()
1619  * @completion: The completion of the write in progress.
1621 static void update_block_map(struct vdo_completion *completion)  in update_block_map()  argument
1623 	struct data_vio *data_vio = reference_count_update_completion_as_data_vio(completion);  in update_block_map()
1633 	completion = &data_vio->vio.completion;  in update_block_map()
1634 	vdo_set_completion_result(completion, data_vio->decrement_completion.result);  in update_block_map()
1635 	if (completion->result != VDO_SUCCESS) {  in update_block_map()
1636 		handle_data_vio_error(completion);  in update_block_map()
1640 	completion->error_handler = handle_data_vio_error;  in update_block_map()
1644 		completion->callback = complete_data_vio;  in update_block_map()
1650 static void decrement_reference_count(struct vdo_completion *completion)  in decrement_reference_count()  argument
1652 	struct data_vio *data_vio = container_of(completion, struct data_vio,  in decrement_reference_count()
1657 	vdo_set_completion_callback(completion, update_block_map,  in decrement_reference_count()
1659 	completion->error_handler = update_block_map;  in decrement_reference_count()
1660 	vdo_modify_reference_count(completion, &data_vio->decrement_updater);  in decrement_reference_count()
1663 static void increment_reference_count(struct vdo_completion *completion)  in increment_reference_count()  argument
1665 	struct data_vio *data_vio = as_data_vio(completion);  in increment_reference_count()
1681 	completion->error_handler = update_block_map;  in increment_reference_count()
1682 	vdo_modify_reference_count(completion, &data_vio->increment_updater);  in increment_reference_count()
1686 static void journal_remapping(struct vdo_completion *completion)  in journal_remapping()  argument
1688 	struct data_vio *data_vio = as_data_vio(completion);  in journal_remapping()
1712 	vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio);  in journal_remapping()
1721 static void read_old_block_mapping(struct vdo_completion *completion)  in read_old_block_mapping()  argument
1723 	struct data_vio *data_vio = as_data_vio(completion);  in read_old_block_mapping()
1749 static void pack_compressed_data(struct vdo_completion *completion)  in pack_compressed_data()  argument
1751 	struct data_vio *data_vio = as_data_vio(completion);  in pack_compressed_data()
1770 static void compress_data_vio(struct vdo_completion *completion)  in compress_data_vio()  argument
1772 	struct data_vio *data_vio = as_data_vio(completion);  in compress_data_vio()
1841 static void hash_data_vio(struct vdo_completion *completion)  in hash_data_vio()  argument
1843 	struct data_vio *data_vio = as_data_vio(completion);  in hash_data_vio()
1880 	vdo_set_completion_result(&data_vio->vio.completion,  in write_bio_finished()
1929 static void acknowledge_write_callback(struct vdo_completion *completion)  in acknowledge_write_callback()  argument
1931 	struct data_vio *data_vio = as_data_vio(completion);  in acknowledge_write_callback()
1932 	struct vdo *vdo = completion->vdo;  in acknowledge_write_callback()
1954 static void allocate_block(struct vdo_completion *completion)  in allocate_block()  argument
1956 	struct data_vio *data_vio = as_data_vio(completion);  in allocate_block()
1963 	completion->error_handler = handle_data_vio_error;  in allocate_block()
1986 static void handle_allocation_error(struct vdo_completion *completion)  in handle_allocation_error()  argument
1988 	struct data_vio *data_vio = as_data_vio(completion);  in handle_allocation_error()
1990 	if (completion->result == VDO_NO_SPACE) {  in handle_allocation_error()
1992 		vdo_reset_completion(completion);  in handle_allocation_error()
1993 		completion->error_handler = handle_data_vio_error;  in handle_allocation_error()
1999 	handle_data_vio_error(completion);  in handle_allocation_error()
2015 void continue_data_vio_with_block_map_slot(struct vdo_completion *completion)  in continue_data_vio_with_block_map_slot()  argument
2017 	struct data_vio *data_vio = as_data_vio(completion);  in continue_data_vio_with_block_map_slot()
2034 		completion->callback = complete_data_vio;  in continue_data_vio_with_block_map_slot()