15 #include "data-vio.h"
19 #include "vio.h"
71  * @vio: The vio associated with the bio.
74 static void count_all_bios(struct vio *vio, struct bio *bio)  in count_all_bios()  argument
76 	struct atomic_statistics *stats = &vio->completion.vdo->stats;  in count_all_bios()
78 	if (is_data_vio(vio)) {  in count_all_bios()
84 	if (vio->type == VIO_TYPE_RECOVERY_JOURNAL)  in count_all_bios()
86 	else if (vio->type == VIO_TYPE_BLOCK_MAP)  in count_all_bios()
91  * assert_in_bio_zone() - Assert that a vio is in the correct bio zone and not in interrupt
93  * @vio: The vio to check.
95 static void assert_in_bio_zone(struct vio *vio)  in assert_in_bio_zone()  argument
98 	assert_vio_in_bio_zone(vio);  in assert_in_bio_zone()
104  * @vio: The vio associated with the bio.
107 static void send_bio_to_device(struct vio *vio, struct bio *bio)  in send_bio_to_device()  argument
109 	struct vdo *vdo = vio->completion.vdo;  in send_bio_to_device()
111 	assert_in_bio_zone(vio);  in send_bio_to_device()
113 	count_all_bios(vio, bio);  in send_bio_to_device()
119  * vdo_submit_vio() - Submits a vio's bio to the underlying block device. May block if the device
124 	struct vio *vio = as_vio(completion);  in vdo_submit_vio()  local
126 	send_bio_to_device(vio, vio->bio);  in vdo_submit_vio()
130  * get_bio_list() - Extract the list of bios to submit from a vio.
131  * @vio: The vio submitting I/O.
133  * The list will always contain at least one entry (the bio for the vio on which it is called), but
138 static struct bio *get_bio_list(struct vio *vio)  in get_bio_list()  argument
141 	struct io_submitter *submitter = vio->completion.vdo->io_submitter;  in get_bio_list()
142 	struct bio_queue_data *bio_queue_data = &(submitter->bio_queue_data[vio->bio_zone]);  in get_bio_list()
144 	assert_in_bio_zone(vio);  in get_bio_list()
148 			   vio->bios_merged.head->bi_iter.bi_sector);  in get_bio_list()
150 			   vio->bios_merged.tail->bi_iter.bi_sector);  in get_bio_list()
151 	bio = vio->bios_merged.head;  in get_bio_list()
152 	bio_list_init(&vio->bios_merged);  in get_bio_list()
167 	struct vio *vio = as_vio(completion);  in submit_data_vio()  local
169 	assert_in_bio_zone(vio);  in submit_data_vio()
170 	for (bio = get_bio_list(vio); bio != NULL; bio = next) {  in submit_data_vio()
173 		send_bio_to_device((struct vio *) bio->bi_private, bio);  in submit_data_vio()
181  * @vio: The vio we want to merge.
187  * Return: the vio to merge to, NULL if no merging is possible.
189 static struct vio *get_mergeable_locked(struct int_map *map, struct vio *vio,  in get_mergeable_locked()  argument
192 	struct bio *bio = vio->bio;  in get_mergeable_locked()
194 	struct vio *vio_merge;  in get_mergeable_locked()
206 	if (vio->completion.priority != vio_merge->completion.priority)  in get_mergeable_locked()
224 static int map_merged_vio(struct int_map *bio_map, struct vio *vio)  in map_merged_vio()  argument
229 	bio_sector = vio->bios_merged.head->bi_iter.bi_sector;  in map_merged_vio()
230 	result = vdo_int_map_put(bio_map, bio_sector, vio, true, NULL);  in map_merged_vio()
234 	bio_sector = vio->bios_merged.tail->bi_iter.bi_sector;  in map_merged_vio()
235 	return vdo_int_map_put(bio_map, bio_sector, vio, true, NULL);  in map_merged_vio()
238 static int merge_to_prev_tail(struct int_map *bio_map, struct vio *vio,  in merge_to_prev_tail()  argument
239 			      struct vio *prev_vio)  in merge_to_prev_tail()
242 	bio_list_merge(&prev_vio->bios_merged, &vio->bios_merged);  in merge_to_prev_tail()
246 static int merge_to_next_head(struct int_map *bio_map, struct vio *vio,  in merge_to_next_head()  argument
247 			      struct vio *next_vio)  in merge_to_next_head()
255 	bio_list_merge_head(&next_vio->bios_merged, &vio->bios_merged);  in merge_to_next_head()
260  * try_bio_map_merge() - Attempt to merge a vio's bio with other pending I/Os.
261  * @vio: The vio to merge.
265  * Return: whether or not the vio was merged.
267 static bool try_bio_map_merge(struct vio *vio)  in try_bio_map_merge()  argument
271 	struct bio *bio = vio->bio;  in try_bio_map_merge()
272 	struct vio *prev_vio, *next_vio;  in try_bio_map_merge()
273 	struct vdo *vdo = vio->completion.vdo;  in try_bio_map_merge()
275 		&vdo->io_submitter->bio_queue_data[vio->bio_zone];  in try_bio_map_merge()
278 	bio_list_init(&vio->bios_merged);  in try_bio_map_merge()
279 	bio_list_add(&vio->bios_merged, bio);  in try_bio_map_merge()
282 	prev_vio = get_mergeable_locked(bio_queue_data->map, vio, true);  in try_bio_map_merge()
283 	next_vio = get_mergeable_locked(bio_queue_data->map, vio, false);  in try_bio_map_merge()
292 					 vio, true, NULL);  in try_bio_map_merge()
295 		result = merge_to_prev_tail(bio_queue_data->map, vio, prev_vio);  in try_bio_map_merge()
298 		result = merge_to_next_head(bio_queue_data->map, vio, next_vio);  in try_bio_map_merge()
316 	if (try_bio_map_merge(&data_vio->vio))  in vdo_submit_data_vio()
323  * __submit_metadata_vio() - Submit I/O for a metadata vio.
324  * @vio: the vio for which to issue I/O
331  * The vio is enqueued on a vdo bio queue so that bio submission (which may block) does not block
339 void __submit_metadata_vio(struct vio *vio, physical_block_number_t physical,  in __submit_metadata_vio()  argument
344 	struct vdo_completion *completion = &vio->completion;  in __submit_metadata_vio()
352 	result = vio_reset_bio(vio, data, callback, operation | REQ_META, physical);  in __submit_metadata_vio()
354 		continue_vio(vio, result);  in __submit_metadata_vio()
359 				    get_vio_bio_zone_thread_id(vio));  in __submit_metadata_vio()
360 	vdo_launch_completion_with_priority(completion, get_metadata_priority(vio));  in __submit_metadata_vio()