19 #include <linux/dm-io.h>
21 #define DM_MSG_PREFIX "io"
31  * Aligning 'struct io' reduces the number of bits required to store
34 struct io {  struct
86  * We need to keep track of which region a bio is doing io for.
88  * ensure the 'struct io' pointer is aligned so enough low bits are
93 static void store_io_and_region_in_bio(struct bio *bio, struct io *io,  in store_io_and_region_in_bio()  argument
96 	if (unlikely(!IS_ALIGNED((unsigned long)io, DM_IO_MAX_REGIONS))) {  in store_io_and_region_in_bio()
97 		DMCRIT("Unaligned struct io pointer %p", io);  in store_io_and_region_in_bio()
101 	bio->bi_private = (void *)((unsigned long)io | region);  in store_io_and_region_in_bio()
104 static void retrieve_io_and_region_from_bio(struct bio *bio, struct io **io,  in retrieve_io_and_region_from_bio()  argument
109 	*io = (void *)(val & -(unsigned long)DM_IO_MAX_REGIONS);  in retrieve_io_and_region_from_bio()
115  * We need an io object to keep track of the number of bios that
116  * have been dispatched for a particular io.
119 static void complete_io(struct io *io)  in complete_io()  argument
121 	unsigned long error_bits = io->error_bits;  in complete_io()
122 	io_notify_fn fn = io->callback;  in complete_io()
123 	void *context = io->context;  in complete_io()
125 	if (io->vma_invalidate_size)  in complete_io()
126 		invalidate_kernel_vmap_range(io->vma_invalidate_address,  in complete_io()
127 					     io->vma_invalidate_size);  in complete_io()
129 	mempool_free(io, &io->client->pool);  in complete_io()
133 static void dec_count(struct io *io, unsigned int region, blk_status_t error)  in dec_count()  argument
136 		set_bit(region, &io->error_bits);  in dec_count()
138 	if (atomic_dec_and_test(&io->count))  in dec_count()
139 		complete_io(io);  in dec_count()
144 	struct io *io;  in endio()  local
152 	 * The bio destructor in bio_put() may use the io object.  in endio()
154 	retrieve_io_and_region_from_bio(bio, &io, &region);  in endio()
159 	dec_count(io, region, error);  in endio()
165  * destination page for io.
303  * IO routines that accept a list of pages.
308 		      struct io *io, unsigned short ioprio)  in do_region()  argument
330 		atomic_inc(&io->count);  in do_region()
331 		dec_count(io, region, BLK_STS_NOTSUPP);  in do_region()
354 				       &io->client->bios);  in do_region()
358 		store_io_and_region_in_bio(bio, io, region);  in do_region()
380 		atomic_inc(&io->count);  in do_region()
387 			struct io *io, unsigned short ioprio)  in dispatch_io()  argument
401 			do_region(opf, i, where + i, dp, io, ioprio);  in dispatch_io()
406 	 * the io being completed too early.  in dispatch_io()
408 	dec_count(io, 0, 0);  in dispatch_io()
416 	struct io *io;  in async_io()  local
418 	io = mempool_alloc(&client->pool, GFP_NOIO);  in async_io()
419 	io->error_bits = 0;  in async_io()
420 	atomic_set(&io->count, 1); /* see dispatch_io() */  in async_io()
421 	io->client = client;  in async_io()
422 	io->callback = fn;  in async_io()
423 	io->context = context;  in async_io()
425 	io->vma_invalidate_address = dp->vma_invalidate_address;  in async_io()
426 	io->vma_invalidate_size = dp->vma_invalidate_size;  in async_io()
428 	dispatch_io(opf, num_regions, where, dp, io, ioprio);  in async_io()
528 	_dm_io_cache = KMEM_CACHE(io, 0);  in dm_io_init()