Lines Matching +full:no +full:- +full:sdio
1 // SPDX-License-Identifier: GPL-2.0-only
3 * fs/direct-io.c
16 * added support for non-aligned IO.
58 * is determined on a per-invocation basis. When talking to the filesystem
60 * down by dio->blkfactor. Similarly, fs-blocksize quantities are converted
75 finer. blkfactor=2 means 1/4-block
77 unsigned start_zero_done; /* flag: sub-blocksize zeroing has
158 static inline unsigned dio_pages_present(struct dio_submit *sdio) in dio_pages_present() argument
160 return sdio->tail - sdio->head; in dio_pages_present()
166 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) in dio_refill_pages() argument
168 struct page **pages = dio->pages; in dio_refill_pages()
169 const enum req_op dio_op = dio->opf & REQ_OP_MASK; in dio_refill_pages()
172 ret = iov_iter_extract_pages(sdio->iter, &pages, LONG_MAX, in dio_refill_pages()
173 DIO_PAGES, 0, &sdio->from); in dio_refill_pages()
175 if (ret < 0 && sdio->blocks_available && dio_op == REQ_OP_WRITE) { in dio_refill_pages()
181 if (dio->page_errors == 0) in dio_refill_pages()
182 dio->page_errors = ret; in dio_refill_pages()
183 dio->pages[0] = ZERO_PAGE(0); in dio_refill_pages()
184 sdio->head = 0; in dio_refill_pages()
185 sdio->tail = 1; in dio_refill_pages()
186 sdio->from = 0; in dio_refill_pages()
187 sdio->to = PAGE_SIZE; in dio_refill_pages()
192 ret += sdio->from; in dio_refill_pages()
193 sdio->head = 0; in dio_refill_pages()
194 sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE; in dio_refill_pages()
195 sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1; in dio_refill_pages()
208 struct dio_submit *sdio) in dio_get_page() argument
210 if (dio_pages_present(sdio) == 0) { in dio_get_page()
213 ret = dio_refill_pages(dio, sdio); in dio_get_page()
216 BUG_ON(dio_pages_present(sdio) == 0); in dio_get_page()
218 return dio->pages[sdio->head]; in dio_get_page()
223 if (dio->is_pinned) in dio_pin_page()
229 if (dio->is_pinned) in dio_unpin_page()
234 * dio_complete() - called when all DIO BIO I/O has been completed
246 const enum req_op dio_op = dio->opf & REQ_OP_MASK; in dio_complete()
247 loff_t offset = dio->iocb->ki_pos; in dio_complete()
254 * In that case -EIOCBQUEUED is in fact not an error we want in dio_complete()
257 if (ret == -EIOCBQUEUED) in dio_complete()
260 if (dio->result) { in dio_complete()
261 transferred = dio->result; in dio_complete()
265 ((offset + transferred) > dio->i_size)) in dio_complete()
266 transferred = dio->i_size - offset; in dio_complete()
268 if (unlikely(ret == -EFAULT) && transferred) in dio_complete()
273 ret = dio->page_errors; in dio_complete()
275 ret = dio->io_error; in dio_complete()
279 if (dio->end_io) { in dio_complete()
281 err = dio->end_io(dio->iocb, offset, ret, dio->private); in dio_complete()
288 * non-direct readahead, or faulted in by get_user_pages() if the source in dio_complete()
293 * And this page cache invalidation has to be after dio->end_io(), as in dio_complete()
300 kiocb_invalidate_post_direct_write(dio->iocb, ret); in dio_complete()
302 inode_dio_end(dio->inode); in dio_complete()
310 dio->iocb->ki_pos += transferred; in dio_complete()
313 ret = generic_write_sync(dio->iocb, ret); in dio_complete()
314 dio->iocb->ki_complete(dio->iocb, ret); in dio_complete()
335 struct dio *dio = bio->bi_private; in dio_bio_end_aio()
336 const enum req_op dio_op = dio->opf & REQ_OP_MASK; in dio_bio_end_aio()
344 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_end_aio()
345 remaining = --dio->refcount; in dio_bio_end_aio()
346 if (remaining == 1 && dio->waiter) in dio_bio_end_aio()
347 wake_up_process(dio->waiter); in dio_bio_end_aio()
348 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_end_aio()
359 if (dio->result) in dio_bio_end_aio()
360 defer_completion = dio->defer_completion || in dio_bio_end_aio()
362 dio->inode->i_mapping->nrpages); in dio_bio_end_aio()
364 INIT_WORK(&dio->complete_work, dio_aio_complete_work); in dio_bio_end_aio()
365 queue_work(dio->inode->i_sb->s_dio_done_wq, in dio_bio_end_aio()
366 &dio->complete_work); in dio_bio_end_aio()
374 * The BIO completion handler simply queues the BIO up for the process-context
378 * implement a singly-linked list of completed BIOs, at dio->bio_list.
382 struct dio *dio = bio->bi_private; in dio_bio_end_io()
385 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_end_io()
386 bio->bi_private = dio->bio_list; in dio_bio_end_io()
387 dio->bio_list = bio; in dio_bio_end_io()
388 if (--dio->refcount == 1 && dio->waiter) in dio_bio_end_io()
389 wake_up_process(dio->waiter); in dio_bio_end_io()
390 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_end_io()
394 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, in dio_bio_alloc() argument
404 bio = bio_alloc(bdev, nr_vecs, dio->opf, GFP_KERNEL); in dio_bio_alloc()
405 bio->bi_iter.bi_sector = first_sector; in dio_bio_alloc()
406 if (dio->is_async) in dio_bio_alloc()
407 bio->bi_end_io = dio_bio_end_aio; in dio_bio_alloc()
409 bio->bi_end_io = dio_bio_end_io; in dio_bio_alloc()
410 if (dio->is_pinned) in dio_bio_alloc()
412 bio->bi_write_hint = file_inode(dio->iocb->ki_filp)->i_write_hint; in dio_bio_alloc()
414 sdio->bio = bio; in dio_bio_alloc()
415 sdio->logical_offset_in_bio = sdio->cur_page_fs_offset; in dio_bio_alloc()
423 * bios hold a dio reference between submit_bio and ->end_io.
425 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) in dio_bio_submit() argument
427 const enum req_op dio_op = dio->opf & REQ_OP_MASK; in dio_bio_submit()
428 struct bio *bio = sdio->bio; in dio_bio_submit()
431 bio->bi_private = dio; in dio_bio_submit()
433 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_submit()
434 dio->refcount++; in dio_bio_submit()
435 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_submit()
437 if (dio->is_async && dio_op == REQ_OP_READ && dio->should_dirty) in dio_bio_submit()
440 dio->bio_disk = bio->bi_bdev->bd_disk; in dio_bio_submit()
444 sdio->bio = NULL; in dio_bio_submit()
445 sdio->boundary = 0; in dio_bio_submit()
446 sdio->logical_offset_in_bio = 0; in dio_bio_submit()
452 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio) in dio_cleanup() argument
454 if (dio->is_pinned) in dio_cleanup()
455 unpin_user_pages(dio->pages + sdio->head, in dio_cleanup()
456 sdio->tail - sdio->head); in dio_cleanup()
457 sdio->head = sdio->tail; in dio_cleanup()
463 * all bios have been issued so that dio->refcount can only decrease. This
471 spin_lock_irqsave(&dio->bio_lock, flags); in dio_await_one()
479 while (dio->refcount > 1 && dio->bio_list == NULL) { in dio_await_one()
481 dio->waiter = current; in dio_await_one()
482 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_await_one()
485 spin_lock_irqsave(&dio->bio_lock, flags); in dio_await_one()
486 dio->waiter = NULL; in dio_await_one()
488 if (dio->bio_list) { in dio_await_one()
489 bio = dio->bio_list; in dio_await_one()
490 dio->bio_list = bio->bi_private; in dio_await_one()
492 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_await_one()
497 * Process one completed BIO. No locks are held.
501 blk_status_t err = bio->bi_status; in dio_bio_complete()
502 const enum req_op dio_op = dio->opf & REQ_OP_MASK; in dio_bio_complete()
503 bool should_dirty = dio_op == REQ_OP_READ && dio->should_dirty; in dio_bio_complete()
506 if (err == BLK_STS_AGAIN && (bio->bi_opf & REQ_NOWAIT)) in dio_bio_complete()
507 dio->io_error = -EAGAIN; in dio_bio_complete()
509 dio->io_error = -EIO; in dio_bio_complete()
512 if (dio->is_async && should_dirty) { in dio_bio_complete()
522 * Wait on and process all in-flight BIOs. This must only be called once
525 * errors are propagated through dio->io_error and should be propagated via
545 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio) in dio_bio_reap() argument
549 if (sdio->reap_counter++ >= 64) { in dio_bio_reap()
550 while (dio->bio_list) { in dio_bio_reap()
555 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_reap()
556 bio = dio->bio_list; in dio_bio_reap()
557 dio->bio_list = bio->bi_private; in dio_bio_reap()
558 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_reap()
563 sdio->reap_counter = 0; in dio_bio_reap()
570 struct super_block *sb = dio->inode->i_sb; in dio_set_defer_completion()
572 if (dio->defer_completion) in dio_set_defer_completion()
574 dio->defer_completion = true; in dio_set_defer_completion()
575 if (!sb->s_dio_done_wq) in dio_set_defer_completion()
582 * of available blocks at sdio->blocks_available. These are in units of the
586 * it uses the passed inode-relative block number as the file offset, as usual.
588 * get_block() is passed the number of i_blkbits-sized blocks which direct_io
591 * If the fs has mapped a lot of blocks, it should populate bh->b_size to
593 * bh->b_blocknr.
598 * In the case of filesystem holes: the fs may return an arbitrarily-large
600 * buffer_mapped(). However the direct-io code will only process holes one
601 * block at a time - it will repeatedly call get_block() as it walks the hole.
603 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, in get_more_blocks() argument
606 const enum req_op dio_op = dio->opf & REQ_OP_MASK; in get_more_blocks()
608 sector_t fs_startblk; /* Into file, in filesystem-sized blocks */ in get_more_blocks()
609 sector_t fs_endblk; /* Into file, in filesystem-sized blocks */ in get_more_blocks()
610 unsigned long fs_count; /* Number of filesystem-sized blocks */ in get_more_blocks()
612 unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor; in get_more_blocks()
619 ret = dio->page_errors; in get_more_blocks()
621 BUG_ON(sdio->block_in_file >= sdio->final_block_in_request); in get_more_blocks()
622 fs_startblk = sdio->block_in_file >> sdio->blkfactor; in get_more_blocks()
623 fs_endblk = (sdio->final_block_in_request - 1) >> in get_more_blocks()
624 sdio->blkfactor; in get_more_blocks()
625 fs_count = fs_endblk - fs_startblk + 1; in get_more_blocks()
627 map_bh->b_state = 0; in get_more_blocks()
628 map_bh->b_size = fs_count << i_blkbits; in get_more_blocks()
642 if (dio->flags & DIO_SKIP_HOLES) { in get_more_blocks()
643 i_size = i_size_read(dio->inode); in get_more_blocks()
644 if (i_size && fs_startblk <= (i_size - 1) >> i_blkbits) in get_more_blocks()
648 ret = (*sdio->get_block)(dio->inode, fs_startblk, in get_more_blocks()
652 dio->private = map_bh->b_private; in get_more_blocks()
661 * There is no bio. Make one now.
663 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio, in dio_new_bio() argument
669 ret = dio_bio_reap(dio, sdio); in dio_new_bio()
672 sector = start_sector << (sdio->blkbits - 9); in dio_new_bio()
673 nr_pages = bio_max_segs(sdio->pages_in_io); in dio_new_bio()
675 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); in dio_new_bio()
676 sdio->boundary = 0; in dio_new_bio()
684 * the just-added page.
686 * Return zero on success. Non-zero means the caller needs to start a new BIO.
688 static inline int dio_bio_add_page(struct dio *dio, struct dio_submit *sdio) in dio_bio_add_page() argument
692 ret = bio_add_page(sdio->bio, sdio->cur_page, in dio_bio_add_page()
693 sdio->cur_page_len, sdio->cur_page_offset); in dio_bio_add_page()
694 if (ret == sdio->cur_page_len) { in dio_bio_add_page()
698 if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE) in dio_bio_add_page()
699 sdio->pages_in_io--; in dio_bio_add_page()
700 dio_pin_page(dio, sdio->cur_page); in dio_bio_add_page()
701 sdio->final_block_in_bio = sdio->cur_page_block + in dio_bio_add_page()
702 (sdio->cur_page_len >> sdio->blkbits); in dio_bio_add_page()
713 * starts on-disk at cur_page_block.
720 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio, in dio_send_cur_page() argument
725 if (sdio->bio) { in dio_send_cur_page()
726 loff_t cur_offset = sdio->cur_page_fs_offset; in dio_send_cur_page()
727 loff_t bio_next_offset = sdio->logical_offset_in_bio + in dio_send_cur_page()
728 sdio->bio->bi_iter.bi_size; in dio_send_cur_page()
733 * Btrfs cannot handle having logically non-contiguous requests in dio_send_cur_page()
736 * Logical: [0-4095][HOLE][8192-12287] in dio_send_cur_page()
737 * Physical: [0-4095] [4096-8191] in dio_send_cur_page()
744 if (sdio->final_block_in_bio != sdio->cur_page_block || in dio_send_cur_page()
746 dio_bio_submit(dio, sdio); in dio_send_cur_page()
749 if (sdio->bio == NULL) { in dio_send_cur_page()
750 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
755 if (dio_bio_add_page(dio, sdio) != 0) { in dio_send_cur_page()
756 dio_bio_submit(dio, sdio); in dio_send_cur_page()
757 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
759 ret = dio_bio_add_page(dio, sdio); in dio_send_cur_page()
775 * The chunk of page starts on-disk at blocknr.
777 * We perform deferred IO, by recording the last-submitted page inside our
785 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, in submit_page_section() argument
789 const enum req_op dio_op = dio->opf & REQ_OP_MASK; in submit_page_section()
791 int boundary = sdio->boundary; /* dio_send_cur_page may clear it */ in submit_page_section()
803 if (sdio->cur_page == page && in submit_page_section()
804 sdio->cur_page_offset + sdio->cur_page_len == offset && in submit_page_section()
805 sdio->cur_page_block + in submit_page_section()
806 (sdio->cur_page_len >> sdio->blkbits) == blocknr) { in submit_page_section()
807 sdio->cur_page_len += len; in submit_page_section()
814 if (sdio->cur_page) { in submit_page_section()
815 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
816 dio_unpin_page(dio, sdio->cur_page); in submit_page_section()
817 sdio->cur_page = NULL; in submit_page_section()
823 sdio->cur_page = page; in submit_page_section()
824 sdio->cur_page_offset = offset; in submit_page_section()
825 sdio->cur_page_len = len; in submit_page_section()
826 sdio->cur_page_block = blocknr; in submit_page_section()
827 sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits; in submit_page_section()
834 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
835 if (sdio->bio) in submit_page_section()
836 dio_bio_submit(dio, sdio); in submit_page_section()
837 dio_unpin_page(dio, sdio->cur_page); in submit_page_section()
838 sdio->cur_page = NULL; in submit_page_section()
845 * the block for us, we need to fill-in the unused portion of the
846 * block with zeros. This happens only if user-buffer, fileoffset or
847 * io length is not filesystem block-size multiple.
852 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio, in dio_zero_block() argument
860 sdio->start_zero_done = 1; in dio_zero_block()
861 if (!sdio->blkfactor || !buffer_new(map_bh)) in dio_zero_block()
864 dio_blocks_per_fs_block = 1 << sdio->blkfactor; in dio_zero_block()
865 this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1); in dio_zero_block()
875 this_chunk_blocks = dio_blocks_per_fs_block - this_chunk_blocks; in dio_zero_block()
877 this_chunk_bytes = this_chunk_blocks << sdio->blkbits; in dio_zero_block()
880 if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes, in dio_zero_block()
881 sdio->next_block_for_io, map_bh)) in dio_zero_block()
884 sdio->next_block_for_io += this_chunk_blocks; in dio_zero_block()
893 * happily perform page-sized but 512-byte aligned IOs. It is important that
896 * So what we do is to permit the ->get_block function to populate bh.b_size
899 * For best results, the blockdev should be set up with 512-byte i_blkbits and
903 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, in do_direct_IO() argument
906 const enum req_op dio_op = dio->opf & REQ_OP_MASK; in do_direct_IO()
907 const unsigned blkbits = sdio->blkbits; in do_direct_IO()
908 const unsigned i_blkbits = blkbits + sdio->blkfactor; in do_direct_IO()
911 while (sdio->block_in_file < sdio->final_block_in_request) { in do_direct_IO()
915 page = dio_get_page(dio, sdio); in do_direct_IO()
920 from = sdio->head ? 0 : sdio->from; in do_direct_IO()
921 to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE; in do_direct_IO()
922 sdio->head++; in do_direct_IO()
929 if (sdio->blocks_available == 0) { in do_direct_IO()
936 ret = get_more_blocks(dio, sdio, map_bh); in do_direct_IO()
944 sdio->blocks_available = in do_direct_IO()
945 map_bh->b_size >> blkbits; in do_direct_IO()
946 sdio->next_block_for_io = in do_direct_IO()
947 map_bh->b_blocknr << sdio->blkfactor; in do_direct_IO()
950 map_bh->b_bdev, in do_direct_IO()
951 map_bh->b_blocknr, in do_direct_IO()
952 map_bh->b_size >> i_blkbits); in do_direct_IO()
955 if (!sdio->blkfactor) in do_direct_IO()
958 blkmask = (1 << sdio->blkfactor) - 1; in do_direct_IO()
959 dio_remainder = (sdio->block_in_file & blkmask); in do_direct_IO()
963 * starts partway into a fs-block, in do_direct_IO()
964 * dio_remainder will be non-zero. If the IO in do_direct_IO()
970 * on-disk in do_direct_IO()
973 sdio->next_block_for_io += dio_remainder; in do_direct_IO()
974 sdio->blocks_available -= dio_remainder; in do_direct_IO()
981 /* AKPM: eargh, -ENOTBLK is a hack */ in do_direct_IO()
984 return -ENOTBLK; in do_direct_IO()
991 i_size_aligned = ALIGN(i_size_read(dio->inode), in do_direct_IO()
993 if (sdio->block_in_file >= in do_direct_IO()
1000 sdio->block_in_file++; in do_direct_IO()
1002 dio->result += 1 << blkbits; in do_direct_IO()
1011 if (unlikely(sdio->blkfactor && !sdio->start_zero_done)) in do_direct_IO()
1012 dio_zero_block(dio, sdio, 0, map_bh); in do_direct_IO()
1018 this_chunk_blocks = sdio->blocks_available; in do_direct_IO()
1019 u = (to - from) >> blkbits; in do_direct_IO()
1022 u = sdio->final_block_in_request - sdio->block_in_file; in do_direct_IO()
1028 if (this_chunk_blocks == sdio->blocks_available) in do_direct_IO()
1029 sdio->boundary = buffer_boundary(map_bh); in do_direct_IO()
1030 ret = submit_page_section(dio, sdio, page, in do_direct_IO()
1033 sdio->next_block_for_io, in do_direct_IO()
1039 sdio->next_block_for_io += this_chunk_blocks; in do_direct_IO()
1041 sdio->block_in_file += this_chunk_blocks; in do_direct_IO()
1043 dio->result += this_chunk_bytes; in do_direct_IO()
1044 sdio->blocks_available -= this_chunk_blocks; in do_direct_IO()
1046 BUG_ON(sdio->block_in_file > sdio->final_block_in_request); in do_direct_IO()
1047 if (sdio->block_in_file == sdio->final_block_in_request) in do_direct_IO()
1068 * return code that the caller will hand to ->complete(). in drop_refcount()
1074 spin_lock_irqsave(&dio->bio_lock, flags); in drop_refcount()
1075 ret2 = --dio->refcount; in drop_refcount()
1076 spin_unlock_irqrestore(&dio->bio_lock, flags); in drop_refcount()
1084 * - if the flags value contains DIO_LOCKING we use a fancy locking
1089 * - if the flags value does NOT contain DIO_LOCKING we don't use any
1100 * NOTE: if you pass "sdio" to anything by pointer make sure that function
1110 unsigned i_blkbits = READ_ONCE(inode->i_blkbits); in __blockdev_direct_IO()
1112 unsigned blocksize_mask = (1 << blkbits) - 1; in __blockdev_direct_IO()
1113 ssize_t retval = -EINVAL; in __blockdev_direct_IO()
1115 loff_t offset = iocb->ki_pos; in __blockdev_direct_IO()
1118 struct dio_submit sdio = { NULL, }; in __blockdev_direct_IO() local
1129 return -ENOMEM; in __blockdev_direct_IO()
1137 dio->flags = flags; in __blockdev_direct_IO()
1138 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) { in __blockdev_direct_IO()
1142 dio->is_pinned = iov_iter_extract_will_pin(iter); in __blockdev_direct_IO()
1145 dio->i_size = i_size_read(inode); in __blockdev_direct_IO()
1146 if (iov_iter_rw(iter) == READ && offset >= dio->i_size) { in __blockdev_direct_IO()
1154 blocksize_mask = (1 << blkbits) - 1; in __blockdev_direct_IO()
1159 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) { in __blockdev_direct_IO()
1160 struct address_space *mapping = iocb->ki_filp->f_mapping; in __blockdev_direct_IO()
1162 retval = filemap_write_and_wait_range(mapping, offset, end - 1); in __blockdev_direct_IO()
1174 dio->is_async = false; in __blockdev_direct_IO()
1176 dio->is_async = false; in __blockdev_direct_IO()
1178 dio->is_async = true; in __blockdev_direct_IO()
1180 dio->inode = inode; in __blockdev_direct_IO()
1182 dio->opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; in __blockdev_direct_IO()
1183 if (iocb->ki_flags & IOCB_NOWAIT) in __blockdev_direct_IO()
1184 dio->opf |= REQ_NOWAIT; in __blockdev_direct_IO()
1186 dio->opf = REQ_OP_READ; in __blockdev_direct_IO()
1191 * so that we can call ->fsync. in __blockdev_direct_IO()
1193 if (dio->is_async && iov_iter_rw(iter) == WRITE) { in __blockdev_direct_IO()
1197 else if (!dio->inode->i_sb->s_dio_done_wq) { in __blockdev_direct_IO()
1203 retval = sb_init_dio_done_wq(dio->inode->i_sb); in __blockdev_direct_IO()
1214 sdio.blkbits = blkbits; in __blockdev_direct_IO()
1215 sdio.blkfactor = i_blkbits - blkbits; in __blockdev_direct_IO()
1216 sdio.block_in_file = offset >> blkbits; in __blockdev_direct_IO()
1218 sdio.get_block = get_block; in __blockdev_direct_IO()
1219 dio->end_io = end_io; in __blockdev_direct_IO()
1220 sdio.final_block_in_bio = -1; in __blockdev_direct_IO()
1221 sdio.next_block_for_io = -1; in __blockdev_direct_IO()
1223 dio->iocb = iocb; in __blockdev_direct_IO()
1225 spin_lock_init(&dio->bio_lock); in __blockdev_direct_IO()
1226 dio->refcount = 1; in __blockdev_direct_IO()
1228 dio->should_dirty = user_backed_iter(iter) && iov_iter_rw(iter) == READ; in __blockdev_direct_IO()
1229 sdio.iter = iter; in __blockdev_direct_IO()
1230 sdio.final_block_in_request = end >> blkbits; in __blockdev_direct_IO()
1233 * In case of non-aligned buffers, we may need 2 more in __blockdev_direct_IO()
1236 if (unlikely(sdio.blkfactor)) in __blockdev_direct_IO()
1237 sdio.pages_in_io = 2; in __blockdev_direct_IO()
1239 sdio.pages_in_io += iov_iter_npages(iter, INT_MAX); in __blockdev_direct_IO()
1243 retval = do_direct_IO(dio, &sdio, &map_bh); in __blockdev_direct_IO()
1245 dio_cleanup(dio, &sdio); in __blockdev_direct_IO()
1247 if (retval == -ENOTBLK) { in __blockdev_direct_IO()
1255 * There may be some unwritten disk at the end of a part-written in __blockdev_direct_IO()
1256 * fs-block-sized block. Go zero that now. in __blockdev_direct_IO()
1258 dio_zero_block(dio, &sdio, 1, &map_bh); in __blockdev_direct_IO()
1260 if (sdio.cur_page) { in __blockdev_direct_IO()
1263 ret2 = dio_send_cur_page(dio, &sdio, &map_bh); in __blockdev_direct_IO()
1266 dio_unpin_page(dio, sdio.cur_page); in __blockdev_direct_IO()
1267 sdio.cur_page = NULL; in __blockdev_direct_IO()
1269 if (sdio.bio) in __blockdev_direct_IO()
1270 dio_bio_submit(dio, &sdio); in __blockdev_direct_IO()
1278 dio_cleanup(dio, &sdio); in __blockdev_direct_IO()
1285 if (iov_iter_rw(iter) == READ && (dio->flags & DIO_LOCKING)) in __blockdev_direct_IO()
1286 inode_unlock(dio->inode); in __blockdev_direct_IO()
1292 * call aio_complete is when we return -EIOCBQUEUED, so we key on that. in __blockdev_direct_IO()
1293 * This had *better* be the only place that raises -EIOCBQUEUED. in __blockdev_direct_IO()
1295 BUG_ON(retval == -EIOCBQUEUED); in __blockdev_direct_IO()
1296 if (dio->is_async && retval == 0 && dio->result && in __blockdev_direct_IO()
1297 (iov_iter_rw(iter) == READ || dio->result == count)) in __blockdev_direct_IO()
1298 retval = -EIOCBQUEUED; in __blockdev_direct_IO()
1305 BUG_ON(retval != -EIOCBQUEUED); in __blockdev_direct_IO()
1310 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) in __blockdev_direct_IO()