Lines Matching full:unlikely
469 if (unlikely(*sec_ptr >= ic->journal_sections)) in wraparound_section()
507 if (unlikely(r < 0)) { in sb_mac()
515 if (unlikely(r < 0)) { in sb_mac()
549 if (unlikely(r)) in sync_rw_sb()
555 if (unlikely(r)) in sync_rw_sb()
561 if (unlikely(r)) in sync_rw_sb()
580 if (unlikely(((sector | n_sectors) & ((1 << ic->sb->log2_sectors_per_block) - 1)) != 0)) { in block_bitmap_op()
590 if (unlikely(!n_sectors)) in block_bitmap_op()
671 if (unlikely(page < end_page)) { in block_bitmap_op()
708 if (unlikely(section >= ic->journal_sections) || in access_journal_check()
709 unlikely(offset >= limit)) { in access_journal_check()
785 if (unlikely(r < 0)) { in section_mac()
794 if (unlikely(r < 0)) { in section_mac()
801 if (unlikely(r < 0)) { in section_mac()
811 if (unlikely(r < 0)) { in section_mac()
821 if (unlikely(r < 0)) { in section_mac()
834 if (unlikely(r < 0)) { in section_mac()
908 while (unlikely(pl_index == section_index)) { in xor_journal()
941 if (unlikely(err)) { in complete_journal_encrypt()
1007 if (unlikely(do_crypt(encrypt, req, comp))) in crypt_journal()
1031 if (unlikely(error != 0)) in complete_journal_io()
1045 if (unlikely(dm_integrity_failed(ic))) { in rw_journal_sectors()
1073 if (unlikely(r)) { in rw_journal_sectors()
1168 if (unlikely(dm_integrity_failed(ic))) { in copy_from_journal()
1190 if (unlikely(r)) { in copy_from_journal()
1213 if (unlikely(ranges_overlap(range, new_range))) in add_new_range()
1241 while (unlikely(!list_empty(&ic->wait_list))) { in remove_range_unlocked()
1277 } while (unlikely(new_range->waiting)); in wait_and_add_new_range()
1282 if (unlikely(!add_new_range(ic, new_range, true))) in add_new_range_and_wait()
1354 if (unlikely(pos >= ic->journal_entries)) in test_journal_node()
1357 if (unlikely(RB_EMPTY_NODE(&node->node))) in test_journal_node()
1359 if (unlikely(node->sector != sector)) in test_journal_node()
1363 if (unlikely(!next)) in test_journal_node()
1379 if (unlikely(!next)) in find_newer_committed_node()
1416 if (unlikely(r)) in dm_integrity_rw_tag()
1436 if (unlikely(memcmp(dp, tag, to_copy))) in dm_integrity_rw_tag()
1437 if (unlikely(!ic->discard) || in dm_integrity_rw_tag()
1438 unlikely(memchr_inv(dp, DISCARD_FILLER, to_copy) != NULL)) { in dm_integrity_rw_tag()
1447 if (unlikely(dp[i] != tag[i])) in dm_integrity_rw_tag()
1452 if (unlikely(hash_offset == ic->tag_size)) { in dm_integrity_rw_tag()
1453 if (unlikely(!may_be)) { in dm_integrity_rw_tag()
1467 if (unlikely(*metadata_offset == 1U << SECTOR_SHIFT << ic->log2_buffer_sectors)) { in dm_integrity_rw_tag()
1472 if (unlikely(!is_power_of_2(ic->tag_size))) in dm_integrity_rw_tag()
1476 } while (unlikely(total_size)); in dm_integrity_rw_tag()
1494 if (unlikely(error != 0)) in flush_notify()
1523 if (unlikely(r)) in dm_integrity_flush_buffers()
1574 if (unlikely(r) && !bio->bi_status) in do_endio()
1576 if (unlikely(ic->synchronous_mode) && bio_op(bio) == REQ_OP_WRITE) { in do_endio()
1592 if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic))) in do_endio_flush()
1606 if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD)) in dec_in_flight()
1610 if (unlikely(dio->bi_status) && !bio->bi_status) in dec_in_flight()
1612 if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) { in dec_in_flight()
1648 if (unlikely(r < 0)) { in integrity_sector_checksum()
1655 if (unlikely(r < 0)) { in integrity_sector_checksum()
1662 if (unlikely(r < 0)) { in integrity_sector_checksum()
1668 if (unlikely(r < 0)) { in integrity_sector_checksum()
1674 if (unlikely(r < 0)) { in integrity_sector_checksum()
1680 if (unlikely(digest_size < ic->tag_size)) in integrity_sector_checksum()
1735 if (unlikely(r)) { in integrity_recheck()
1782 unsigned int extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; in integrity_metadata()
1787 if (unlikely(ic->mode == 'R')) in integrity_metadata()
1804 if (unlikely(dio->op == REQ_OP_DISCARD)) { in integrity_metadata()
1817 if (unlikely(r)) { in integrity_metadata()
1854 if (unlikely(r)) { in integrity_metadata()
1867 if (unlikely(pos < bv_copy.bv_len)) { in integrity_metadata()
1896 if (unlikely(r)) in integrity_metadata()
1914 if (unlikely(logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) { in dm_integrity_check_limits()
1920 if (unlikely((logical_sector | bio_sectors(bio)) & (unsigned int)(ic->sectors_per_block - 1))) { in dm_integrity_check_limits()
1931 if (unlikely(bv.bv_len & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) { in dm_integrity_check_limits()
1961 if (unlikely(dio->op == REQ_OP_DISCARD)) { in dm_integrity_map()
1976 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) { in dm_integrity_map()
1983 if (unlikely(dio->fua)) { in dm_integrity_map()
1990 if (unlikely(!dm_integrity_check_limits(ic, dio->range.logical_sector, bio))) in dm_integrity_map()
2002 if (unlikely(wanted_tag_size != bip->bip_iter.bi_size)) { in dm_integrity_map()
2009 if (unlikely(bip != NULL)) { in dm_integrity_map()
2015 if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ)) in dm_integrity_map()
2039 if (unlikely(bv.bv_len >> SECTOR_SHIFT > n_sectors)) in __journal_read_write()
2051 if (unlikely(dio->op == REQ_OP_READ)) { in __journal_read_write()
2056 if (unlikely(journal_entry_is_inprogress(je))) { in __journal_read_write()
2079 if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) { in __journal_read_write()
2109 } while (unlikely(tag_todo)); in __journal_read_write()
2129 if (unlikely(digest_size > ic->tag_size)) { in __journal_read_write()
2143 if (unlikely(journal_entry == ic->journal_section_entries)) { in __journal_read_write()
2152 if (unlikely(dio->op == REQ_OP_READ)) in __journal_read_write()
2159 if (unlikely(waitqueue_active(&ic->copy_to_journal_wait))) in __journal_read_write()
2168 if (unlikely(bio->bi_iter.bi_size)) { in __journal_read_write()
2191 if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D') in dm_integrity_map_continue()
2203 if (unlikely(dm_integrity_failed(ic))) { in dm_integrity_map_continue()
2217 if (unlikely(!dio->range.n_sectors)) { in dm_integrity_map_continue()
2243 if (unlikely(pos >= ic->journal_entries)) in dm_integrity_map_continue()
2250 if (unlikely(we == ic->journal_section_entries)) { in dm_integrity_map_continue()
2264 if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector)) in dm_integrity_map_continue()
2278 if (unlikely(!add_new_range(ic, &dio->range, true))) { in dm_integrity_map_continue()
2304 if (unlikely(new_pos != journal_read_pos)) { in dm_integrity_map_continue()
2315 if (unlikely(new_pos != NOT_FOUND) || in dm_integrity_map_continue()
2316 unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) { in dm_integrity_map_continue()
2330 if (unlikely(journal_read_pos != NOT_FOUND)) { in dm_integrity_map_continue()
2336 if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) { in dm_integrity_map_continue()
2365 if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) { in dm_integrity_map_continue()
2403 if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry))) in dm_integrity_map_continue()
2417 if (unlikely(bio_integrity(bio))) { in dm_integrity_map_inline()
2424 if (unlikely((bio->bi_opf & REQ_PREFLUSH) != 0)) in dm_integrity_map_inline()
2432 extra_size = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; in dm_integrity_map_inline()
2435 if (unlikely(!dio->integrity_payload)) { in dm_integrity_map_inline()
2470 if (unlikely(!add_new_range(ic, &dio->range, true))) { in dm_integrity_map_inline()
2484 if (unlikely(!dio->integrity_payload)) { in dm_integrity_map_inline()
2491 if (unlikely(!dm_integrity_check_limits(ic, bio->bi_iter.bi_sector, bio))) { in dm_integrity_map_inline()
2520 if (unlikely(ret != dio->payload_len)) { in dm_integrity_map_inline()
2532 if (unlikely(dio->integrity_payload_from_mempool)) in dm_integrity_free_payload()
2563 if (unlikely(r != (ic->sectors_per_block << SECTOR_SHIFT))) { in dm_integrity_inline_recheck()
2579 if (unlikely(r != ic->tuple_size)) { in dm_integrity_inline_recheck()
2589 if (unlikely(r != 0)) { in dm_integrity_inline_recheck()
2598 …if (unlikely(memcmp(digest, dio->integrity_payload, min(crypto_shash_digestsize(ic->internal_hash)… in dm_integrity_inline_recheck()
2629 unlikely(dio->integrity_range_locked)) in dm_integrity_end_io()
2637 if (unlikely(memcmp(digest, dio->integrity_payload + pos, in dm_integrity_end_io()
2652 if (unlikely(dio->integrity_range_locked)) in dm_integrity_end_io()
2718 if (unlikely(ic->mode != 'J')) { in integrity_commit()
2749 if (unlikely(i >= ic->journal_sections)) in integrity_commit()
2785 if (unlikely(error != 0)) in complete_copy_from_journal()
2817 if (unlikely(from_replay)) in do_journal_write()
2830 BUG_ON(unlikely(journal_entry_is_inprogress(je)) && !from_replay); in do_journal_write()
2832 if (unlikely(from_replay)) { in do_journal_write()
2833 if (unlikely(sec & (unsigned int)(ic->sectors_per_block - 1))) { in do_journal_write()
2837 if (unlikely(sec >= ic->provided_data_sectors)) { in do_journal_write()
2850 BUG_ON(unlikely(journal_entry_is_inprogress(je2)) && !from_replay); in do_journal_write()
2852 if (unlikely(sec2 >= ic->provided_data_sectors)) in do_journal_write()
2907 unlikely(from_replay) && in do_journal_write()
2914 if (unlikely(memcmp(test_tag, journal_entry_tag(ic, je2), ic->tag_size))) { in do_journal_write()
2923 if (unlikely(r)) in do_journal_write()
2971 if (unlikely(!prev_free_sectors)) in integrity_writer()
2986 if (unlikely(r)) in recalc_write_super()
3035 if (unlikely(dm_post_suspending(ic->ti))) in integrity_recalc()
3039 if (unlikely(range.logical_sector >= ic->provided_data_sectors)) { in integrity_recalc()
3078 if (unlikely(++super_counter == RECALC_WRITE_SUPER)) { in integrity_recalc()
3086 if (unlikely(dm_integrity_failed(ic))) in integrity_recalc()
3099 if (unlikely(r)) { in integrity_recalc()
3113 if (unlikely(r)) { in integrity_recalc()
3192 if (unlikely(dm_post_suspending(ic->ti))) in integrity_recalc_inline()
3196 if (unlikely(range.logical_sector >= ic->provided_data_sectors)) in integrity_recalc_inline()
3203 if (unlikely(++super_counter == RECALC_WRITE_SUPER)) { in integrity_recalc_inline()
3208 if (unlikely(dm_integrity_failed(ic))) in integrity_recalc_inline()
3218 if (unlikely(r)) { in integrity_recalc_inline()
3235 if (unlikely(IS_ERR(bip))) { in integrity_recalc_inline()
3241 if (unlikely(ret != t - recalc_tags)) { in integrity_recalc_inline()
3249 if (unlikely(r)) { in integrity_recalc_inline()
3365 while (unlikely((bio = bio_list_pop(&ic->synchronous_bios)) != NULL)) { in bitmap_flush_work()
3503 if (unlikely(write_start >= ic->journal_sections)) in replay_journal()
3524 if (unlikely(i >= ic->journal_sections)) in replay_journal()
3642 if (unlikely(r)) in dm_integrity_postsuspend()
3676 if (unlikely(r)) in dm_integrity_resume()
3718 if (unlikely(r)) in dm_integrity_resume()
3730 if (unlikely(r)) in dm_integrity_resume()
5119 if (unlikely(r)) { in dm_integrity_ctr()