Lines Matching +full:super +full:- +full:set
1 // SPDX-License-Identifier: GPL-2.0-only
8 #include "dm-zoned.h"
22 * On-disk super block magic.
30 * On disk super block.
35 * (1) Super block (1 block)
54 /* The number of metadata blocks, including this super block */
72 /* DM-Zoned label */
75 /* DM-Zoned UUID */
99 * Chunk mapping table metadata: 512 8-bytes entries per 4KB block.
103 #define DMZ_MAP_ENTRIES_MASK (DMZ_MAP_ENTRIES - 1)
130 * Super block information (one per metadata set).
141 * In-memory metadata.
208 DMINFO("(%s): " format, (zmd)->label, ## args)
211 DMERR("(%s): " format, (zmd)->label, ## args)
214 DMWARN("(%s): " format, (zmd)->label, ## args)
217 DMDEBUG("(%s): " format, (zmd)->label, ## args)
226 return zone->id - zone->dev->zone_offset; in dmz_dev_zone_id()
233 return (sector_t)zone_id << zmd->zone_nr_sectors_shift; in dmz_start_sect()
240 return (sector_t)zone_id << zmd->zone_nr_blocks_shift; in dmz_start_block()
245 return zmd->zone_nr_blocks; in dmz_zone_nr_blocks()
250 return zmd->zone_nr_blocks_shift; in dmz_zone_nr_blocks_shift()
255 return zmd->zone_nr_sectors; in dmz_zone_nr_sectors()
260 return zmd->zone_nr_sectors_shift; in dmz_zone_nr_sectors_shift()
265 return zmd->nr_zones; in dmz_nr_zones()
270 return zmd->nr_chunks; in dmz_nr_chunks()
275 return zmd->dev[idx].nr_rnd; in dmz_nr_rnd_zones()
280 return atomic_read(&zmd->dev[idx].unmap_nr_rnd); in dmz_nr_unmap_rnd_zones()
285 return zmd->nr_cache; in dmz_nr_cache_zones()
290 return atomic_read(&zmd->unmap_nr_cache); in dmz_nr_unmap_cache_zones()
295 return zmd->dev[idx].nr_seq; in dmz_nr_seq_zones()
300 return atomic_read(&zmd->dev[idx].unmap_nr_seq); in dmz_nr_unmap_seq_zones()
305 return xa_load(&zmd->zones, zone_id); in dmz_get()
314 return ERR_PTR(-ENOMEM); in dmz_insert()
316 if (xa_insert(&zmd->zones, zone_id, zone, GFP_KERNEL)) { in dmz_insert()
318 return ERR_PTR(-EBUSY); in dmz_insert()
321 INIT_LIST_HEAD(&zone->link); in dmz_insert()
322 atomic_set(&zone->refcount, 0); in dmz_insert()
323 zone->id = zone_id; in dmz_insert()
324 zone->chunk = DMZ_MAP_UNMAPPED; in dmz_insert()
325 zone->dev = dev; in dmz_insert()
332 return (const char *)zmd->label; in dmz_metadata_label()
339 for (i = 0; i < zmd->nr_devs; i++) { in dmz_check_dev()
340 if (!dmz_check_bdev(&zmd->dev[i])) in dmz_check_dev()
350 for (i = 0; i < zmd->nr_devs; i++) { in dmz_dev_is_dying()
351 if (dmz_bdev_is_dying(&zmd->dev[i])) in dmz_dev_is_dying()
363 mutex_lock(&zmd->map_lock); in dmz_lock_map()
368 mutex_unlock(&zmd->map_lock); in dmz_unlock_map()
380 down_read(&zmd->mblk_sem); in dmz_lock_metadata()
385 up_read(&zmd->mblk_sem); in dmz_unlock_metadata()
395 mutex_lock(&zmd->mblk_flush_lock); in dmz_lock_flush()
400 mutex_unlock(&zmd->mblk_flush_lock); in dmz_unlock_flush()
412 if (zmd->max_nr_mblks && atomic_read(&zmd->nr_mblks) > zmd->max_nr_mblks) { in dmz_alloc_mblock()
413 spin_lock(&zmd->mblk_lock); in dmz_alloc_mblock()
414 mblk = list_first_entry_or_null(&zmd->mblk_lru_list, in dmz_alloc_mblock()
417 list_del_init(&mblk->link); in dmz_alloc_mblock()
418 rb_erase(&mblk->node, &zmd->mblk_rbtree); in dmz_alloc_mblock()
419 mblk->no = mblk_no; in dmz_alloc_mblock()
421 spin_unlock(&zmd->mblk_lock); in dmz_alloc_mblock()
431 mblk->page = alloc_page(GFP_NOIO); in dmz_alloc_mblock()
432 if (!mblk->page) { in dmz_alloc_mblock()
437 RB_CLEAR_NODE(&mblk->node); in dmz_alloc_mblock()
438 INIT_LIST_HEAD(&mblk->link); in dmz_alloc_mblock()
439 mblk->ref = 0; in dmz_alloc_mblock()
440 mblk->state = 0; in dmz_alloc_mblock()
441 mblk->no = mblk_no; in dmz_alloc_mblock()
442 mblk->data = page_address(mblk->page); in dmz_alloc_mblock()
444 atomic_inc(&zmd->nr_mblks); in dmz_alloc_mblock()
454 __free_pages(mblk->page, 0); in dmz_free_mblock()
457 atomic_dec(&zmd->nr_mblks); in dmz_free_mblock()
465 struct rb_root *root = &zmd->mblk_rbtree; in dmz_insert_mblock()
466 struct rb_node **new = &(root->rb_node), *parent = NULL; in dmz_insert_mblock()
473 new = (b->no < mblk->no) ? &((*new)->rb_left) : &((*new)->rb_right); in dmz_insert_mblock()
477 rb_link_node(&mblk->node, parent, new); in dmz_insert_mblock()
478 rb_insert_color(&mblk->node, root); in dmz_insert_mblock()
488 struct rb_root *root = &zmd->mblk_rbtree; in dmz_get_mblock_fast()
489 struct rb_node *node = root->rb_node; in dmz_get_mblock_fast()
494 if (mblk->no == mblk_no) { in dmz_get_mblock_fast()
499 mblk->ref++; in dmz_get_mblock_fast()
500 if (mblk->ref == 1 && in dmz_get_mblock_fast()
501 !test_bit(DMZ_META_DIRTY, &mblk->state)) in dmz_get_mblock_fast()
502 list_del_init(&mblk->link); in dmz_get_mblock_fast()
505 node = (mblk->no < mblk_no) ? node->rb_left : node->rb_right; in dmz_get_mblock_fast()
516 struct dmz_mblock *mblk = bio->bi_private; in dmz_mblock_bio_end_io()
519 if (bio->bi_status) in dmz_mblock_bio_end_io()
520 set_bit(DMZ_META_ERROR, &mblk->state); in dmz_mblock_bio_end_io()
527 clear_bit_unlock(flag, &mblk->state); in dmz_mblock_bio_end_io()
529 wake_up_bit(&mblk->state, flag); in dmz_mblock_bio_end_io()
541 sector_t block = zmd->sb[zmd->mblk_primary].block + mblk_no; in dmz_get_mblock_slow()
542 struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev; in dmz_get_mblock_slow()
546 return ERR_PTR(-EIO); in dmz_get_mblock_slow()
551 return ERR_PTR(-ENOMEM); in dmz_get_mblock_slow()
553 bio = bio_alloc(dev->bdev, 1, REQ_OP_READ | REQ_META | REQ_PRIO, in dmz_get_mblock_slow()
556 spin_lock(&zmd->mblk_lock); in dmz_get_mblock_slow()
564 spin_unlock(&zmd->mblk_lock); in dmz_get_mblock_slow()
570 mblk->ref++; in dmz_get_mblock_slow()
571 set_bit(DMZ_META_READING, &mblk->state); in dmz_get_mblock_slow()
574 spin_unlock(&zmd->mblk_lock); in dmz_get_mblock_slow()
577 bio->bi_iter.bi_sector = dmz_blk2sect(block); in dmz_get_mblock_slow()
578 bio->bi_private = mblk; in dmz_get_mblock_slow()
579 bio->bi_end_io = dmz_mblock_bio_end_io; in dmz_get_mblock_slow()
580 __bio_add_page(bio, mblk->page, DMZ_BLOCK_SIZE, 0); in dmz_get_mblock_slow()
595 if (!zmd->max_nr_mblks) in dmz_shrink_mblock_cache()
598 while (!list_empty(&zmd->mblk_lru_list) && in dmz_shrink_mblock_cache()
599 atomic_read(&zmd->nr_mblks) > zmd->min_nr_mblks && in dmz_shrink_mblock_cache()
601 mblk = list_first_entry(&zmd->mblk_lru_list, in dmz_shrink_mblock_cache()
603 list_del_init(&mblk->link); in dmz_shrink_mblock_cache()
604 rb_erase(&mblk->node, &zmd->mblk_rbtree); in dmz_shrink_mblock_cache()
618 struct dmz_metadata *zmd = shrink->private_data; in dmz_mblock_shrinker_count()
620 return atomic_read(&zmd->nr_mblks); in dmz_mblock_shrinker_count()
629 struct dmz_metadata *zmd = shrink->private_data; in dmz_mblock_shrinker_scan()
632 spin_lock(&zmd->mblk_lock); in dmz_mblock_shrinker_scan()
633 count = dmz_shrink_mblock_cache(zmd, sc->nr_to_scan); in dmz_mblock_shrinker_scan()
634 spin_unlock(&zmd->mblk_lock); in dmz_mblock_shrinker_scan()
649 spin_lock(&zmd->mblk_lock); in dmz_release_mblock()
651 mblk->ref--; in dmz_release_mblock()
652 if (mblk->ref == 0) { in dmz_release_mblock()
653 if (test_bit(DMZ_META_ERROR, &mblk->state)) { in dmz_release_mblock()
654 rb_erase(&mblk->node, &zmd->mblk_rbtree); in dmz_release_mblock()
656 } else if (!test_bit(DMZ_META_DIRTY, &mblk->state)) { in dmz_release_mblock()
657 list_add_tail(&mblk->link, &zmd->mblk_lru_list); in dmz_release_mblock()
662 spin_unlock(&zmd->mblk_lock); in dmz_release_mblock()
673 struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev; in dmz_get_mblock()
676 spin_lock(&zmd->mblk_lock); in dmz_get_mblock()
678 spin_unlock(&zmd->mblk_lock); in dmz_get_mblock()
687 /* Wait for on-going read I/O and check for error */ in dmz_get_mblock()
688 wait_on_bit_io(&mblk->state, DMZ_META_READING, in dmz_get_mblock()
690 if (test_bit(DMZ_META_ERROR, &mblk->state)) { in dmz_get_mblock()
693 return ERR_PTR(-EIO); in dmz_get_mblock()
704 spin_lock(&zmd->mblk_lock); in dmz_dirty_mblock()
705 if (!test_and_set_bit(DMZ_META_DIRTY, &mblk->state)) in dmz_dirty_mblock()
706 list_add_tail(&mblk->link, &zmd->mblk_dirty_list); in dmz_dirty_mblock()
707 spin_unlock(&zmd->mblk_lock); in dmz_dirty_mblock()
714 unsigned int set) in dmz_write_mblock() argument
716 struct dmz_dev *dev = zmd->sb[set].dev; in dmz_write_mblock()
717 sector_t block = zmd->sb[set].block + mblk->no; in dmz_write_mblock()
721 return -EIO; in dmz_write_mblock()
723 bio = bio_alloc(dev->bdev, 1, REQ_OP_WRITE | REQ_META | REQ_PRIO, in dmz_write_mblock()
726 set_bit(DMZ_META_WRITING, &mblk->state); in dmz_write_mblock()
728 bio->bi_iter.bi_sector = dmz_blk2sect(block); in dmz_write_mblock()
729 bio->bi_private = mblk; in dmz_write_mblock()
730 bio->bi_end_io = dmz_mblock_bio_end_io; in dmz_write_mblock()
731 __bio_add_page(bio, mblk->page, DMZ_BLOCK_SIZE, 0); in dmz_write_mblock()
747 return -EIO; in dmz_rdwr_block()
750 return -EIO; in dmz_rdwr_block()
752 bio = bio_alloc(dev->bdev, 1, op | REQ_SYNC | REQ_META | REQ_PRIO, in dmz_rdwr_block()
754 bio->bi_iter.bi_sector = dmz_blk2sect(block); in dmz_rdwr_block()
765 * Write super block of the specified metadata set.
767 static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set) in dmz_write_sb() argument
769 struct dmz_mblock *mblk = zmd->sb[set].mblk; in dmz_write_sb()
770 struct dmz_super *sb = zmd->sb[set].sb; in dmz_write_sb()
771 struct dmz_dev *dev = zmd->sb[set].dev; in dmz_write_sb()
773 u64 sb_gen = zmd->sb_gen + 1; in dmz_write_sb()
776 sb->magic = cpu_to_le32(DMZ_MAGIC); in dmz_write_sb()
778 sb->version = cpu_to_le32(zmd->sb_version); in dmz_write_sb()
779 if (zmd->sb_version > 1) { in dmz_write_sb()
781 export_uuid(sb->dmz_uuid, &zmd->uuid); in dmz_write_sb()
782 memcpy(sb->dmz_label, zmd->label, BDEVNAME_SIZE); in dmz_write_sb()
783 export_uuid(sb->dev_uuid, &dev->uuid); in dmz_write_sb()
786 sb->gen = cpu_to_le64(sb_gen); in dmz_write_sb()
790 * ie relative to the entire block range, not the per-device in dmz_write_sb()
793 sb_block = zmd->sb[set].zone->id << zmd->zone_nr_blocks_shift; in dmz_write_sb()
794 sb->sb_block = cpu_to_le64(sb_block); in dmz_write_sb()
795 sb->nr_meta_blocks = cpu_to_le32(zmd->nr_meta_blocks); in dmz_write_sb()
796 sb->nr_reserved_seq = cpu_to_le32(zmd->nr_reserved_seq); in dmz_write_sb()
797 sb->nr_chunks = cpu_to_le32(zmd->nr_chunks); in dmz_write_sb()
799 sb->nr_map_blocks = cpu_to_le32(zmd->nr_map_blocks); in dmz_write_sb()
800 sb->nr_bitmap_blocks = cpu_to_le32(zmd->nr_bitmap_blocks); in dmz_write_sb()
802 sb->crc = 0; in dmz_write_sb()
803 sb->crc = cpu_to_le32(crc32_le(sb_gen, (unsigned char *)sb, DMZ_BLOCK_SIZE)); in dmz_write_sb()
805 ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block, in dmz_write_sb()
806 mblk->page); in dmz_write_sb()
808 ret = blkdev_issue_flush(dev->bdev); in dmz_write_sb()
814 * Write dirty metadata blocks to the specified set.
818 unsigned int set) in dmz_write_dirty_mblocks() argument
821 struct dmz_dev *dev = zmd->sb[set].dev; in dmz_write_dirty_mblocks()
828 ret = dmz_write_mblock(zmd, mblk, set); in dmz_write_dirty_mblocks()
839 wait_on_bit_io(&mblk->state, DMZ_META_WRITING, in dmz_write_dirty_mblocks()
841 if (test_bit(DMZ_META_ERROR, &mblk->state)) { in dmz_write_dirty_mblocks()
842 clear_bit(DMZ_META_ERROR, &mblk->state); in dmz_write_dirty_mblocks()
844 ret = -EIO; in dmz_write_dirty_mblocks()
846 nr_mblks_submitted--; in dmz_write_dirty_mblocks()
851 ret = blkdev_issue_flush(dev->bdev); in dmz_write_dirty_mblocks()
862 unsigned int log_set = zmd->mblk_primary ^ 0x1; in dmz_log_dirty_mblocks()
872 * log index super block generation. in dmz_log_dirty_mblocks()
901 down_write(&zmd->mblk_sem); in dmz_flush_metadata()
902 dev = zmd->sb[zmd->mblk_primary].dev; in dmz_flush_metadata()
911 ret = -EIO; in dmz_flush_metadata()
916 spin_lock(&zmd->mblk_lock); in dmz_flush_metadata()
917 list_splice_init(&zmd->mblk_dirty_list, &write_list); in dmz_flush_metadata()
918 spin_unlock(&zmd->mblk_lock); in dmz_flush_metadata()
922 ret = blkdev_issue_flush(dev->bdev); in dmz_flush_metadata()
927 * The primary metadata set is still clean. Keep it this way until in dmz_flush_metadata()
928 * all updates are successful in the secondary set. That is, use in dmz_flush_metadata()
929 * the secondary set as a log. in dmz_flush_metadata()
937 * in the primary metadata set. in dmz_flush_metadata()
939 ret = dmz_write_dirty_mblocks(zmd, &write_list, zmd->mblk_primary); in dmz_flush_metadata()
943 ret = dmz_write_sb(zmd, zmd->mblk_primary); in dmz_flush_metadata()
949 list_del_init(&mblk->link); in dmz_flush_metadata()
951 spin_lock(&zmd->mblk_lock); in dmz_flush_metadata()
952 clear_bit(DMZ_META_DIRTY, &mblk->state); in dmz_flush_metadata()
953 if (mblk->ref == 0) in dmz_flush_metadata()
954 list_add_tail(&mblk->link, &zmd->mblk_lru_list); in dmz_flush_metadata()
955 spin_unlock(&zmd->mblk_lock); in dmz_flush_metadata()
958 zmd->sb_gen++; in dmz_flush_metadata()
961 up_write(&zmd->mblk_sem); in dmz_flush_metadata()
967 spin_lock(&zmd->mblk_lock); in dmz_flush_metadata()
968 list_splice(&write_list, &zmd->mblk_dirty_list); in dmz_flush_metadata()
969 spin_unlock(&zmd->mblk_lock); in dmz_flush_metadata()
972 ret = -EIO; in dmz_flush_metadata()
977 * Check super block.
982 struct dmz_super *sb = dsb->sb; in dmz_check_sb()
983 struct dmz_dev *dev = dsb->dev; in dmz_check_sb()
988 if (le32_to_cpu(sb->magic) != DMZ_MAGIC) { in dmz_check_sb()
990 DMZ_MAGIC, le32_to_cpu(sb->magic)); in dmz_check_sb()
991 return -ENXIO; in dmz_check_sb()
994 zmd->sb_version = le32_to_cpu(sb->version); in dmz_check_sb()
995 if (zmd->sb_version > DMZ_META_VER) { in dmz_check_sb()
997 DMZ_META_VER, zmd->sb_version); in dmz_check_sb()
998 return -EINVAL; in dmz_check_sb()
1000 if (zmd->sb_version < 2 && tertiary) { in dmz_check_sb()
1002 return -EINVAL; in dmz_check_sb()
1005 gen = le64_to_cpu(sb->gen); in dmz_check_sb()
1006 stored_crc = le32_to_cpu(sb->crc); in dmz_check_sb()
1007 sb->crc = 0; in dmz_check_sb()
1012 return -ENXIO; in dmz_check_sb()
1015 sb_block = le64_to_cpu(sb->sb_block); in dmz_check_sb()
1016 if (sb_block != (u64)dsb->zone->id << zmd->zone_nr_blocks_shift) { in dmz_check_sb()
1018 sb_block, (u64)dsb->zone->id << zmd->zone_nr_blocks_shift); in dmz_check_sb()
1019 return -EINVAL; in dmz_check_sb()
1021 if (zmd->sb_version > 1) { in dmz_check_sb()
1024 import_uuid(&sb_uuid, sb->dmz_uuid); in dmz_check_sb()
1026 dmz_dev_err(dev, "NULL DM-Zoned uuid"); in dmz_check_sb()
1027 return -ENXIO; in dmz_check_sb()
1028 } else if (uuid_is_null(&zmd->uuid)) { in dmz_check_sb()
1029 uuid_copy(&zmd->uuid, &sb_uuid); in dmz_check_sb()
1030 } else if (!uuid_equal(&zmd->uuid, &sb_uuid)) { in dmz_check_sb()
1031 dmz_dev_err(dev, "mismatching DM-Zoned uuid, is %pUl expected %pUl", in dmz_check_sb()
1032 &sb_uuid, &zmd->uuid); in dmz_check_sb()
1033 return -ENXIO; in dmz_check_sb()
1035 if (!strlen(zmd->label)) in dmz_check_sb()
1036 memcpy(zmd->label, sb->dmz_label, BDEVNAME_SIZE); in dmz_check_sb()
1037 else if (memcmp(zmd->label, sb->dmz_label, BDEVNAME_SIZE)) { in dmz_check_sb()
1038 dmz_dev_err(dev, "mismatching DM-Zoned label, is %s expected %s", in dmz_check_sb()
1039 sb->dmz_label, zmd->label); in dmz_check_sb()
1040 return -ENXIO; in dmz_check_sb()
1042 import_uuid(&dev->uuid, sb->dev_uuid); in dmz_check_sb()
1043 if (uuid_is_null(&dev->uuid)) { in dmz_check_sb()
1045 return -ENXIO; in dmz_check_sb()
1060 nr_meta_zones = (le32_to_cpu(sb->nr_meta_blocks) + zmd->zone_nr_blocks - 1) in dmz_check_sb()
1061 >> zmd->zone_nr_blocks_shift; in dmz_check_sb()
1063 (zmd->nr_devs <= 1 && nr_meta_zones >= zmd->nr_rnd_zones) || in dmz_check_sb()
1064 (zmd->nr_devs > 1 && nr_meta_zones >= zmd->nr_cache_zones)) { in dmz_check_sb()
1066 return -ENXIO; in dmz_check_sb()
1069 if (!le32_to_cpu(sb->nr_reserved_seq) || in dmz_check_sb()
1070 le32_to_cpu(sb->nr_reserved_seq) >= (zmd->nr_useable_zones - nr_meta_zones)) { in dmz_check_sb()
1072 return -ENXIO; in dmz_check_sb()
1075 nr_data_zones = zmd->nr_useable_zones - in dmz_check_sb()
1076 (nr_meta_zones * 2 + le32_to_cpu(sb->nr_reserved_seq)); in dmz_check_sb()
1077 if (le32_to_cpu(sb->nr_chunks) > nr_data_zones) { in dmz_check_sb()
1079 le32_to_cpu(sb->nr_chunks), nr_data_zones); in dmz_check_sb()
1080 return -ENXIO; in dmz_check_sb()
1084 zmd->nr_meta_blocks = le32_to_cpu(sb->nr_meta_blocks); in dmz_check_sb()
1085 zmd->nr_reserved_seq = le32_to_cpu(sb->nr_reserved_seq); in dmz_check_sb()
1086 zmd->nr_chunks = le32_to_cpu(sb->nr_chunks); in dmz_check_sb()
1087 zmd->nr_map_blocks = le32_to_cpu(sb->nr_map_blocks); in dmz_check_sb()
1088 zmd->nr_bitmap_blocks = le32_to_cpu(sb->nr_bitmap_blocks); in dmz_check_sb()
1089 zmd->nr_meta_zones = nr_meta_zones; in dmz_check_sb()
1090 zmd->nr_data_zones = nr_data_zones; in dmz_check_sb()
1096 * Read the first or second super block from disk.
1098 static int dmz_read_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set) in dmz_read_sb() argument
1100 dmz_zmd_debug(zmd, "read superblock set %d dev %pg block %llu", in dmz_read_sb()
1101 set, sb->dev->bdev, sb->block); in dmz_read_sb()
1103 return dmz_rdwr_block(sb->dev, REQ_OP_READ, in dmz_read_sb()
1104 sb->block, sb->mblk->page); in dmz_read_sb()
1108 * Determine the position of the secondary super blocks on disk.
1109 * This is used only if a corruption of the primary super block
1114 unsigned int zone_nr_blocks = zmd->zone_nr_blocks; in dmz_lookup_secondary_sb()
1116 unsigned int zone_id = zmd->sb[0].zone->id; in dmz_lookup_secondary_sb()
1122 return -ENOMEM; in dmz_lookup_secondary_sb()
1124 zmd->sb[1].mblk = mblk; in dmz_lookup_secondary_sb()
1125 zmd->sb[1].sb = mblk->data; in dmz_lookup_secondary_sb()
1127 /* Bad first super block: search for the second one */ in dmz_lookup_secondary_sb()
1128 zmd->sb[1].block = zmd->sb[0].block + zone_nr_blocks; in dmz_lookup_secondary_sb()
1129 zmd->sb[1].zone = dmz_get(zmd, zone_id + 1); in dmz_lookup_secondary_sb()
1130 zmd->sb[1].dev = zmd->sb[0].dev; in dmz_lookup_secondary_sb()
1131 for (i = 1; i < zmd->nr_rnd_zones; i++) { in dmz_lookup_secondary_sb()
1132 if (dmz_read_sb(zmd, &zmd->sb[1], 1) != 0) in dmz_lookup_secondary_sb()
1134 if (le32_to_cpu(zmd->sb[1].sb->magic) == DMZ_MAGIC) in dmz_lookup_secondary_sb()
1136 zmd->sb[1].block += zone_nr_blocks; in dmz_lookup_secondary_sb()
1137 zmd->sb[1].zone = dmz_get(zmd, zone_id + i); in dmz_lookup_secondary_sb()
1141 zmd->sb[1].mblk = NULL; in dmz_lookup_secondary_sb()
1142 zmd->sb[1].zone = NULL; in dmz_lookup_secondary_sb()
1143 zmd->sb[1].dev = NULL; in dmz_lookup_secondary_sb()
1145 return -EIO; in dmz_lookup_secondary_sb()
1149 * Read a super block from disk.
1151 static int dmz_get_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set) in dmz_get_sb() argument
1159 return -ENOMEM; in dmz_get_sb()
1161 sb->mblk = mblk; in dmz_get_sb()
1162 sb->sb = mblk->data; in dmz_get_sb()
1164 /* Read super block */ in dmz_get_sb()
1165 ret = dmz_read_sb(zmd, sb, set); in dmz_get_sb()
1168 sb->mblk = NULL; in dmz_get_sb()
1176 * Recover a metadata set.
1184 dmz_dev_warn(zmd->sb[dst_set].dev, in dmz_recover_mblocks()
1185 "Metadata set %u invalid: recovering", dst_set); in dmz_recover_mblocks()
1188 zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone); in dmz_recover_mblocks()
1190 zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone); in dmz_recover_mblocks()
1194 return -ENOMEM; in dmz_recover_mblocks()
1197 for (i = 1; i < zmd->nr_meta_blocks; i++) { in dmz_recover_mblocks()
1198 ret = dmz_rdwr_block(zmd->sb[src_set].dev, REQ_OP_READ, in dmz_recover_mblocks()
1199 zmd->sb[src_set].block + i, page); in dmz_recover_mblocks()
1202 ret = dmz_rdwr_block(zmd->sb[dst_set].dev, REQ_OP_WRITE, in dmz_recover_mblocks()
1203 zmd->sb[dst_set].block + i, page); in dmz_recover_mblocks()
1208 /* Finalize with the super block */ in dmz_recover_mblocks()
1209 if (!zmd->sb[dst_set].mblk) { in dmz_recover_mblocks()
1210 zmd->sb[dst_set].mblk = dmz_alloc_mblock(zmd, 0); in dmz_recover_mblocks()
1211 if (!zmd->sb[dst_set].mblk) { in dmz_recover_mblocks()
1212 ret = -ENOMEM; in dmz_recover_mblocks()
1215 zmd->sb[dst_set].sb = zmd->sb[dst_set].mblk->data; in dmz_recover_mblocks()
1226 * Get super block from disk.
1234 if (!zmd->sb[0].zone) { in dmz_load_sb()
1235 dmz_zmd_err(zmd, "Primary super block zone not set"); in dmz_load_sb()
1236 return -ENXIO; in dmz_load_sb()
1239 /* Read and check the primary super block */ in dmz_load_sb()
1240 zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone); in dmz_load_sb()
1241 zmd->sb[0].dev = zmd->sb[0].zone->dev; in dmz_load_sb()
1242 ret = dmz_get_sb(zmd, &zmd->sb[0], 0); in dmz_load_sb()
1244 dmz_dev_err(zmd->sb[0].dev, "Read primary super block failed"); in dmz_load_sb()
1248 ret = dmz_check_sb(zmd, &zmd->sb[0], false); in dmz_load_sb()
1250 /* Read and check secondary super block */ in dmz_load_sb()
1253 if (!zmd->sb[1].zone) { in dmz_load_sb()
1255 zmd->sb[0].zone->id + zmd->nr_meta_zones; in dmz_load_sb()
1257 zmd->sb[1].zone = dmz_get(zmd, zone_id); in dmz_load_sb()
1259 zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone); in dmz_load_sb()
1260 zmd->sb[1].dev = zmd->sb[0].dev; in dmz_load_sb()
1261 ret = dmz_get_sb(zmd, &zmd->sb[1], 1); in dmz_load_sb()
1266 dmz_dev_err(zmd->sb[1].dev, "Read secondary super block failed"); in dmz_load_sb()
1270 ret = dmz_check_sb(zmd, &zmd->sb[1], false); in dmz_load_sb()
1276 dmz_zmd_err(zmd, "No valid super block found"); in dmz_load_sb()
1277 return -EIO; in dmz_load_sb()
1281 sb_gen[0] = le64_to_cpu(zmd->sb[0].sb->gen); in dmz_load_sb()
1285 dmz_dev_err(zmd->sb[0].dev, in dmz_load_sb()
1287 return -EIO; in dmz_load_sb()
1292 sb_gen[1] = le64_to_cpu(zmd->sb[1].sb->gen); in dmz_load_sb()
1297 dmz_dev_err(zmd->sb[1].dev, in dmz_load_sb()
1299 return -EIO; in dmz_load_sb()
1304 zmd->sb_gen = sb_gen[0]; in dmz_load_sb()
1305 zmd->mblk_primary = 0; in dmz_load_sb()
1307 zmd->sb_gen = sb_gen[1]; in dmz_load_sb()
1308 zmd->mblk_primary = 1; in dmz_load_sb()
1311 dmz_dev_debug(zmd->sb[zmd->mblk_primary].dev, in dmz_load_sb()
1312 "Using super block %u (gen %llu)", in dmz_load_sb()
1313 zmd->mblk_primary, zmd->sb_gen); in dmz_load_sb()
1315 if (zmd->sb_version > 1) { in dmz_load_sb()
1321 return -ENOMEM; in dmz_load_sb()
1322 for (i = 1; i < zmd->nr_devs; i++) { in dmz_load_sb()
1323 sb->block = 0; in dmz_load_sb()
1324 sb->zone = dmz_get(zmd, zmd->dev[i].zone_offset); in dmz_load_sb()
1325 sb->dev = &zmd->dev[i]; in dmz_load_sb()
1326 if (!dmz_is_meta(sb->zone)) { in dmz_load_sb()
1327 dmz_dev_err(sb->dev, in dmz_load_sb()
1328 "Tertiary super block zone %u not marked as metadata zone", in dmz_load_sb()
1329 sb->zone->id); in dmz_load_sb()
1330 ret = -EINVAL; in dmz_load_sb()
1335 dmz_dev_err(sb->dev, in dmz_load_sb()
1336 "Read tertiary super block failed"); in dmz_load_sb()
1337 dmz_free_mblock(zmd, sb->mblk); in dmz_load_sb()
1341 dmz_free_mblock(zmd, sb->mblk); in dmz_load_sb()
1342 if (ret == -EINVAL) in dmz_load_sb()
1357 struct dmz_metadata *zmd = dev->metadata; in dmz_init_zone()
1358 int idx = num + dev->zone_offset; in dmz_init_zone()
1365 if (blkz->len != zmd->zone_nr_sectors) { in dmz_init_zone()
1366 if (zmd->sb_version > 1) { in dmz_init_zone()
1368 set_bit(DMZ_OFFLINE, &zone->flags); in dmz_init_zone()
1370 } else if (blkz->start + blkz->len == dev->capacity) in dmz_init_zone()
1372 return -ENXIO; in dmz_init_zone()
1379 if (blkz->capacity != blkz->len) in dmz_init_zone()
1380 return -ENXIO; in dmz_init_zone()
1382 switch (blkz->type) { in dmz_init_zone()
1384 set_bit(DMZ_RND, &zone->flags); in dmz_init_zone()
1388 set_bit(DMZ_SEQ, &zone->flags); in dmz_init_zone()
1391 return -ENXIO; in dmz_init_zone()
1395 zone->wp_block = 0; in dmz_init_zone()
1397 zone->wp_block = dmz_sect2blk(blkz->wp - blkz->start); in dmz_init_zone()
1399 if (blkz->cond == BLK_ZONE_COND_OFFLINE) in dmz_init_zone()
1400 set_bit(DMZ_OFFLINE, &zone->flags); in dmz_init_zone()
1401 else if (blkz->cond == BLK_ZONE_COND_READONLY) in dmz_init_zone()
1402 set_bit(DMZ_READ_ONLY, &zone->flags); in dmz_init_zone()
1404 zmd->nr_useable_zones++; in dmz_init_zone()
1406 zmd->nr_rnd_zones++; in dmz_init_zone()
1407 if (zmd->nr_devs == 1 && !zmd->sb[0].zone) { in dmz_init_zone()
1408 /* Primary super block zone */ in dmz_init_zone()
1409 zmd->sb[0].zone = zone; in dmz_init_zone()
1412 if (zmd->nr_devs > 1 && num == 0) { in dmz_init_zone()
1418 set_bit(DMZ_META, &zone->flags); in dmz_init_zone()
1429 for (idx = 0; idx < dev->nr_zones; idx++) { in dmz_emulate_zones()
1435 set_bit(DMZ_CACHE, &zone->flags); in dmz_emulate_zones()
1436 zone->wp_block = 0; in dmz_emulate_zones()
1437 zmd->nr_cache_zones++; in dmz_emulate_zones()
1438 zmd->nr_useable_zones++; in dmz_emulate_zones()
1439 if (dev->capacity - zone_offset < zmd->zone_nr_sectors) { in dmz_emulate_zones()
1441 set_bit(DMZ_OFFLINE, &zone->flags); in dmz_emulate_zones()
1444 zone_offset += zmd->zone_nr_sectors; in dmz_emulate_zones()
1456 for (idx = 0; idx < zmd->nr_zones; idx++) { in dmz_drop_zones()
1457 struct dm_zone *zone = xa_load(&zmd->zones, idx); in dmz_drop_zones()
1460 xa_erase(&zmd->zones, idx); in dmz_drop_zones()
1462 xa_destroy(&zmd->zones); in dmz_drop_zones()
1472 struct dmz_dev *zoned_dev = &zmd->dev[0]; in dmz_init_zones()
1475 zmd->zone_nr_sectors = zmd->dev[0].zone_nr_sectors; in dmz_init_zones()
1476 zmd->zone_nr_sectors_shift = ilog2(zmd->zone_nr_sectors); in dmz_init_zones()
1477 zmd->zone_nr_blocks = dmz_sect2blk(zmd->zone_nr_sectors); in dmz_init_zones()
1478 zmd->zone_nr_blocks_shift = ilog2(zmd->zone_nr_blocks); in dmz_init_zones()
1479 zmd->zone_bitmap_size = zmd->zone_nr_blocks >> 3; in dmz_init_zones()
1480 zmd->zone_nr_bitmap_blocks = in dmz_init_zones()
1481 max_t(sector_t, 1, zmd->zone_bitmap_size >> DMZ_BLOCK_SHIFT); in dmz_init_zones()
1482 zmd->zone_bits_per_mblk = min_t(sector_t, zmd->zone_nr_blocks, in dmz_init_zones()
1486 zmd->nr_zones = 0; in dmz_init_zones()
1487 for (i = 0; i < zmd->nr_devs; i++) { in dmz_init_zones()
1488 struct dmz_dev *dev = &zmd->dev[i]; in dmz_init_zones()
1490 dev->metadata = zmd; in dmz_init_zones()
1491 zmd->nr_zones += dev->nr_zones; in dmz_init_zones()
1493 atomic_set(&dev->unmap_nr_rnd, 0); in dmz_init_zones()
1494 INIT_LIST_HEAD(&dev->unmap_rnd_list); in dmz_init_zones()
1495 INIT_LIST_HEAD(&dev->map_rnd_list); in dmz_init_zones()
1497 atomic_set(&dev->unmap_nr_seq, 0); in dmz_init_zones()
1498 INIT_LIST_HEAD(&dev->unmap_seq_list); in dmz_init_zones()
1499 INIT_LIST_HEAD(&dev->map_seq_list); in dmz_init_zones()
1502 if (!zmd->nr_zones) { in dmz_init_zones()
1503 DMERR("(%s): No zones found", zmd->devname); in dmz_init_zones()
1504 return -ENXIO; in dmz_init_zones()
1506 xa_init(&zmd->zones); in dmz_init_zones()
1509 zmd->devname, sizeof(struct dm_zone) * zmd->nr_zones); in dmz_init_zones()
1511 if (zmd->nr_devs > 1) { in dmz_init_zones()
1512 ret = dmz_emulate_zones(zmd, &zmd->dev[0]); in dmz_init_zones()
1515 zmd->devname, ret); in dmz_init_zones()
1524 zmd->sb[0].zone = dmz_get(zmd, 0); in dmz_init_zones()
1526 for (i = 1; i < zmd->nr_devs; i++) { in dmz_init_zones()
1527 zoned_dev = &zmd->dev[i]; in dmz_init_zones()
1529 ret = blkdev_report_zones(zoned_dev->bdev, 0, in dmz_init_zones()
1534 zmd->devname, ret); in dmz_init_zones()
1544 * time, determine where the super block should be: first block of the in dmz_init_zones()
1547 ret = blkdev_report_zones(zoned_dev->bdev, 0, BLK_ALL_ZONES, in dmz_init_zones()
1551 zmd->devname, ret); in dmz_init_zones()
1564 clear_bit(DMZ_OFFLINE, &zone->flags); in dmz_update_zone_cb()
1565 clear_bit(DMZ_READ_ONLY, &zone->flags); in dmz_update_zone_cb()
1566 if (blkz->cond == BLK_ZONE_COND_OFFLINE) in dmz_update_zone_cb()
1567 set_bit(DMZ_OFFLINE, &zone->flags); in dmz_update_zone_cb()
1568 else if (blkz->cond == BLK_ZONE_COND_READONLY) in dmz_update_zone_cb()
1569 set_bit(DMZ_READ_ONLY, &zone->flags); in dmz_update_zone_cb()
1572 zone->wp_block = dmz_sect2blk(blkz->wp - blkz->start); in dmz_update_zone_cb()
1574 zone->wp_block = 0; in dmz_update_zone_cb()
1583 struct dmz_dev *dev = zone->dev; in dmz_update_zone()
1587 if (dev->flags & DMZ_BDEV_REGULAR) in dmz_update_zone()
1592 * GFP_KERNEL by default for memory allocations, set the per-task in dmz_update_zone()
1597 ret = blkdev_report_zones(dev->bdev, dmz_start_sect(zmd, zone), 1, in dmz_update_zone()
1602 ret = -EIO; in dmz_update_zone()
1605 zone->id); in dmz_update_zone()
1620 struct dmz_dev *dev = zone->dev; in dmz_handle_seq_write_err()
1624 wp = zone->wp_block; in dmz_handle_seq_write_err()
1630 zone->id, zone->wp_block, wp); in dmz_handle_seq_write_err()
1632 if (zone->wp_block < wp) { in dmz_handle_seq_write_err()
1633 dmz_invalidate_blocks(zmd, zone, zone->wp_block, in dmz_handle_seq_write_err()
1634 wp - zone->wp_block); in dmz_handle_seq_write_err()
1657 struct dmz_dev *dev = zone->dev; in dmz_reset_zone()
1661 ret = blkdev_zone_mgmt(dev->bdev, REQ_OP_ZONE_RESET, in dmz_reset_zone()
1663 zmd->zone_nr_sectors); in dmz_reset_zone()
1667 zone->id, ret); in dmz_reset_zone()
1673 clear_bit(DMZ_SEQ_WRITE_ERR, &zone->flags); in dmz_reset_zone()
1674 zone->wp_block = 0; in dmz_reset_zone()
1694 zmd->map_mblk = kcalloc(zmd->nr_map_blocks, in dmz_load_mapping()
1696 if (!zmd->map_mblk) in dmz_load_mapping()
1697 return -ENOMEM; in dmz_load_mapping()
1700 while (chunk < zmd->nr_chunks) { in dmz_load_mapping()
1706 zmd->map_mblk[i] = dmap_mblk; in dmz_load_mapping()
1707 dmap = dmap_mblk->data; in dmz_load_mapping()
1717 if (dzone_id >= zmd->nr_zones) { in dmz_load_mapping()
1720 return -EIO; in dmz_load_mapping()
1727 return -EIO; in dmz_load_mapping()
1729 set_bit(DMZ_DATA, &dzone->flags); in dmz_load_mapping()
1730 dzone->chunk = chunk; in dmz_load_mapping()
1734 list_add_tail(&dzone->link, &zmd->map_cache_list); in dmz_load_mapping()
1736 list_add_tail(&dzone->link, &dzone->dev->map_rnd_list); in dmz_load_mapping()
1738 list_add_tail(&dzone->link, &dzone->dev->map_seq_list); in dmz_load_mapping()
1745 if (bzone_id >= zmd->nr_zones) { in dmz_load_mapping()
1748 return -EIO; in dmz_load_mapping()
1755 return -EIO; in dmz_load_mapping()
1760 return -EIO; in dmz_load_mapping()
1763 set_bit(DMZ_DATA, &bzone->flags); in dmz_load_mapping()
1764 set_bit(DMZ_BUF, &bzone->flags); in dmz_load_mapping()
1765 bzone->chunk = chunk; in dmz_load_mapping()
1766 bzone->bzone = dzone; in dmz_load_mapping()
1767 dzone->bzone = bzone; in dmz_load_mapping()
1770 list_add_tail(&bzone->link, &zmd->map_cache_list); in dmz_load_mapping()
1772 list_add_tail(&bzone->link, &bzone->dev->map_rnd_list); in dmz_load_mapping()
1785 for (i = 0; i < zmd->nr_zones; i++) { in dmz_load_mapping()
1795 zmd->nr_cache++; in dmz_load_mapping()
1797 dzone->dev->nr_rnd++; in dmz_load_mapping()
1799 dzone->dev->nr_seq++; in dmz_load_mapping()
1807 set_bit(DMZ_DATA, &dzone->flags); in dmz_load_mapping()
1808 dzone->chunk = DMZ_MAP_UNMAPPED; in dmz_load_mapping()
1810 list_add_tail(&dzone->link, &zmd->unmap_cache_list); in dmz_load_mapping()
1811 atomic_inc(&zmd->unmap_nr_cache); in dmz_load_mapping()
1813 list_add_tail(&dzone->link, in dmz_load_mapping()
1814 &dzone->dev->unmap_rnd_list); in dmz_load_mapping()
1815 atomic_inc(&dzone->dev->unmap_nr_rnd); in dmz_load_mapping()
1816 } else if (atomic_read(&zmd->nr_reserved_seq_zones) < zmd->nr_reserved_seq) { in dmz_load_mapping()
1817 list_add_tail(&dzone->link, &zmd->reserved_seq_zones_list); in dmz_load_mapping()
1818 set_bit(DMZ_RESERVED, &dzone->flags); in dmz_load_mapping()
1819 atomic_inc(&zmd->nr_reserved_seq_zones); in dmz_load_mapping()
1820 dzone->dev->nr_seq--; in dmz_load_mapping()
1822 list_add_tail(&dzone->link, in dmz_load_mapping()
1823 &dzone->dev->unmap_seq_list); in dmz_load_mapping()
1824 atomic_inc(&dzone->dev->unmap_nr_seq); in dmz_load_mapping()
1832 * Set a data chunk mapping.
1837 struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT]; in dmz_set_chunk_mapping()
1838 struct dmz_map *dmap = dmap_mblk->data; in dmz_set_chunk_mapping()
1852 if (list_empty(&zone->link)) in __dmz_lru_zone()
1855 list_del_init(&zone->link); in __dmz_lru_zone()
1858 list_add_tail(&zone->link, &zone->dev->map_seq_list); in __dmz_lru_zone()
1861 list_add_tail(&zone->link, &zmd->map_cache_list); in __dmz_lru_zone()
1864 list_add_tail(&zone->link, &zone->dev->map_rnd_list); in __dmz_lru_zone()
1875 if (zone->bzone) in dmz_lru_zone()
1876 __dmz_lru_zone(zmd, zone->bzone); in dmz_lru_zone()
1886 prepare_to_wait(&zmd->free_wq, &wait, TASK_UNINTERRUPTIBLE); in dmz_wait_for_free_zones()
1894 finish_wait(&zmd->free_wq, &wait); in dmz_wait_for_free_zones()
1898 * Lock a zone for reclaim (set the zone RECLAIM bit).
1908 return !test_and_set_bit(DMZ_RECLAIM, &zone->flags); in dmz_lock_zone_reclaim()
1919 clear_bit_unlock(DMZ_RECLAIM, &zone->flags); in dmz_unlock_zone_reclaim()
1921 wake_up_bit(&zone->flags, DMZ_RECLAIM); in dmz_unlock_zone_reclaim()
1931 set_bit(DMZ_RECLAIM_TERMINATE, &zone->flags); in dmz_wait_for_reclaim()
1932 wait_on_bit_timeout(&zone->flags, DMZ_RECLAIM, TASK_UNINTERRUPTIBLE, HZ); in dmz_wait_for_reclaim()
1933 clear_bit(DMZ_RECLAIM_TERMINATE, &zone->flags); in dmz_wait_for_reclaim()
1949 if (zmd->nr_cache) { in dmz_get_rnd_zone_for_reclaim()
1950 zone_list = &zmd->map_cache_list; in dmz_get_rnd_zone_for_reclaim()
1953 zone_list = &zmd->dev[idx].map_rnd_list; in dmz_get_rnd_zone_for_reclaim()
1955 zone_list = &zmd->dev[idx].map_rnd_list; in dmz_get_rnd_zone_for_reclaim()
1963 dzone = zone->bzone; in dmz_get_rnd_zone_for_reclaim()
1964 if (dmz_is_rnd(dzone) && dzone->dev->dev_idx != idx) in dmz_get_rnd_zone_for_reclaim()
1966 if (!maxw_z || maxw_z->weight < dzone->weight) in dmz_get_rnd_zone_for_reclaim()
1985 dzone = zone->bzone; in dmz_get_rnd_zone_for_reclaim()
1986 if (dmz_is_rnd(dzone) && dzone->dev->dev_idx != idx) in dmz_get_rnd_zone_for_reclaim()
2005 list_for_each_entry(zone, &zmd->dev[idx].map_seq_list, link) { in dmz_get_seq_zone_for_reclaim()
2006 if (!zone->bzone) in dmz_get_seq_zone_for_reclaim()
2032 if (list_empty(&zmd->reserved_seq_zones_list)) in dmz_get_zone_for_reclaim()
2045 * The zone returned will be set to the active state.
2050 struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT]; in dmz_get_chunk_mapping()
2051 struct dmz_map *dmap = dmap_mblk->data; in dmz_get_chunk_mapping()
2056 int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND; in dmz_get_chunk_mapping()
2074 dzone = ERR_PTR(-EIO); in dmz_get_chunk_mapping()
2087 dzone = ERR_PTR(-EIO); in dmz_get_chunk_mapping()
2090 if (dzone->chunk != chunk) { in dmz_get_chunk_mapping()
2091 dzone = ERR_PTR(-EIO); in dmz_get_chunk_mapping()
2099 dzone = ERR_PTR(-EIO); in dmz_get_chunk_mapping()
2102 clear_bit(DMZ_SEQ_WRITE_ERR, &dzone->flags); in dmz_get_chunk_mapping()
2135 bzone = dzone->bzone; in dmz_put_chunk_mapping()
2168 int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND; in dmz_get_chunk_buffer()
2172 bzone = dzone->bzone; in dmz_get_chunk_buffer()
2180 bzone = ERR_PTR(-EIO); in dmz_get_chunk_buffer()
2188 dmz_set_chunk_mapping(zmd, dzone->chunk, dzone->id, bzone->id); in dmz_get_chunk_buffer()
2190 set_bit(DMZ_BUF, &bzone->flags); in dmz_get_chunk_buffer()
2191 bzone->chunk = dzone->chunk; in dmz_get_chunk_buffer()
2192 bzone->bzone = dzone; in dmz_get_chunk_buffer()
2193 dzone->bzone = bzone; in dmz_get_chunk_buffer()
2195 list_add_tail(&bzone->link, &zmd->map_cache_list); in dmz_get_chunk_buffer()
2197 list_add_tail(&bzone->link, &bzone->dev->map_rnd_list); in dmz_get_chunk_buffer()
2217 for (i = 0; i < zmd->nr_devs; i++) in dmz_alloc_zone()
2218 dmz_schedule_reclaim(zmd->dev[i].reclaim); in dmz_alloc_zone()
2224 list = &zmd->unmap_cache_list; in dmz_alloc_zone()
2226 list = &zmd->dev[dev_idx].unmap_rnd_list; in dmz_alloc_zone()
2228 list = &zmd->dev[dev_idx].unmap_seq_list; in dmz_alloc_zone()
2239 if (i < zmd->nr_devs) { in dmz_alloc_zone()
2240 dev_idx = (dev_idx + 1) % zmd->nr_devs; in dmz_alloc_zone()
2248 zone = list_first_entry_or_null(&zmd->reserved_seq_zones_list, in dmz_alloc_zone()
2251 list_del_init(&zone->link); in dmz_alloc_zone()
2252 atomic_dec(&zmd->nr_reserved_seq_zones); in dmz_alloc_zone()
2258 list_del_init(&zone->link); in dmz_alloc_zone()
2261 atomic_dec(&zmd->unmap_nr_cache); in dmz_alloc_zone()
2263 atomic_dec(&zone->dev->unmap_nr_rnd); in dmz_alloc_zone()
2265 atomic_dec(&zone->dev->unmap_nr_seq); in dmz_alloc_zone()
2268 dmz_zmd_warn(zmd, "Zone %u is offline", zone->id); in dmz_alloc_zone()
2273 dmz_zmd_warn(zmd, "Zone %u has metadata", zone->id); in dmz_alloc_zone()
2292 list_add_tail(&zone->link, &zmd->unmap_cache_list); in dmz_free_zone()
2293 atomic_inc(&zmd->unmap_nr_cache); in dmz_free_zone()
2295 list_add_tail(&zone->link, &zone->dev->unmap_rnd_list); in dmz_free_zone()
2296 atomic_inc(&zone->dev->unmap_nr_rnd); in dmz_free_zone()
2298 list_add_tail(&zone->link, &zmd->reserved_seq_zones_list); in dmz_free_zone()
2299 atomic_inc(&zmd->nr_reserved_seq_zones); in dmz_free_zone()
2301 list_add_tail(&zone->link, &zone->dev->unmap_seq_list); in dmz_free_zone()
2302 atomic_inc(&zone->dev->unmap_nr_seq); in dmz_free_zone()
2305 wake_up_all(&zmd->free_wq); in dmz_free_zone()
2315 /* Set the chunk mapping */ in dmz_map_zone()
2316 dmz_set_chunk_mapping(zmd, chunk, dzone->id, in dmz_map_zone()
2318 dzone->chunk = chunk; in dmz_map_zone()
2320 list_add_tail(&dzone->link, &zmd->map_cache_list); in dmz_map_zone()
2322 list_add_tail(&dzone->link, &dzone->dev->map_rnd_list); in dmz_map_zone()
2324 list_add_tail(&dzone->link, &dzone->dev->map_seq_list); in dmz_map_zone()
2333 unsigned int chunk = zone->chunk; in dmz_unmap_zone()
2341 if (test_and_clear_bit(DMZ_BUF, &zone->flags)) { in dmz_unmap_zone()
2346 dzone_id = zone->bzone->id; in dmz_unmap_zone()
2347 zone->bzone->bzone = NULL; in dmz_unmap_zone()
2348 zone->bzone = NULL; in dmz_unmap_zone()
2355 if (WARN_ON(zone->bzone)) { in dmz_unmap_zone()
2356 zone->bzone->bzone = NULL; in dmz_unmap_zone()
2357 zone->bzone = NULL; in dmz_unmap_zone()
2364 zone->chunk = DMZ_MAP_UNMAPPED; in dmz_unmap_zone()
2365 list_del_init(&zone->link); in dmz_unmap_zone()
2369 * Set @nr_bits bits in @bitmap starting from @bit.
2380 if (((bit & (BITS_PER_LONG - 1)) == 0) && in dmz_set_bits()
2381 ((end - bit) >= BITS_PER_LONG)) { in dmz_set_bits()
2382 /* Try to set the whole word at once */ in dmz_set_bits()
2407 sector_t bitmap_block = 1 + zmd->nr_map_blocks + in dmz_get_bitmap()
2408 (sector_t)(zone->id * zmd->zone_nr_bitmap_blocks) + in dmz_get_bitmap()
2424 while (chunk_block < zmd->zone_nr_blocks) { in dmz_copy_valid_blocks()
2434 memcpy(to_mblk->data, from_mblk->data, DMZ_BLOCK_SIZE); in dmz_copy_valid_blocks()
2440 chunk_block += zmd->zone_bits_per_mblk; in dmz_copy_valid_blocks()
2443 to_zone->weight = from_zone->weight; in dmz_copy_valid_blocks()
2459 while (chunk_block < zmd->zone_nr_blocks) { in dmz_merge_valid_blocks()
2477 * Validate all the blocks in the range [block..block+nr_blocks-1].
2483 unsigned int zone_nr_blocks = zmd->zone_nr_blocks; in dmz_validate_blocks()
2488 zone->id, (unsigned long long)chunk_block, in dmz_validate_blocks()
2499 /* Set bits */ in dmz_validate_blocks()
2501 nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit); in dmz_validate_blocks()
2503 count = dmz_set_bits((unsigned long *)mblk->data, bit, nr_bits); in dmz_validate_blocks()
2510 nr_blocks -= nr_bits; in dmz_validate_blocks()
2514 if (likely(zone->weight + n <= zone_nr_blocks)) in dmz_validate_blocks()
2515 zone->weight += n; in dmz_validate_blocks()
2518 zone->id, zone->weight, in dmz_validate_blocks()
2519 zone_nr_blocks - n); in dmz_validate_blocks()
2520 zone->weight = zone_nr_blocks; in dmz_validate_blocks()
2537 if (((bit & (BITS_PER_LONG - 1)) == 0) && in dmz_clear_bits()
2538 ((end - bit) >= BITS_PER_LONG)) { in dmz_clear_bits()
2558 * Invalidate all the blocks in the range [block..block+nr_blocks-1].
2568 zone->id, (u64)chunk_block, nr_blocks); in dmz_invalidate_blocks()
2570 WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks); in dmz_invalidate_blocks()
2580 nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit); in dmz_invalidate_blocks()
2582 count = dmz_clear_bits((unsigned long *)mblk->data, in dmz_invalidate_blocks()
2590 nr_blocks -= nr_bits; in dmz_invalidate_blocks()
2594 if (zone->weight >= n) in dmz_invalidate_blocks()
2595 zone->weight -= n; in dmz_invalidate_blocks()
2598 zone->id, zone->weight, n); in dmz_invalidate_blocks()
2599 zone->weight = 0; in dmz_invalidate_blocks()
2614 WARN_ON(chunk_block >= zmd->zone_nr_blocks); in dmz_test_block()
2623 (unsigned long *) mblk->data) != 0; in dmz_test_block()
2632 * value specified by set. Search at most nr_blocks blocks from chunk_block.
2636 int set) in dmz_to_next_set_block() argument
2640 unsigned int zone_bits = zmd->zone_bits_per_mblk; in dmz_to_next_set_block()
2644 WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks); in dmz_to_next_set_block()
2653 bitmap = (unsigned long *) mblk->data; in dmz_to_next_set_block()
2655 nr_bits = min(nr_blocks, zone_bits - bit); in dmz_to_next_set_block()
2656 if (set) in dmz_to_next_set_block()
2662 n += set_bit - bit; in dmz_to_next_set_block()
2666 nr_blocks -= nr_bits; in dmz_to_next_set_block()
2688 zmd->zone_nr_blocks - chunk_block, 0); in dmz_block_valid()
2704 zmd->zone_nr_blocks - start_block, 1); in dmz_first_valid_block()
2712 zmd->zone_nr_blocks - start_block, 0); in dmz_first_valid_block()
2716 * Count the number of bits set starting from bit up to bit + nr_bits - 1.
2725 if (((bit & (BITS_PER_LONG - 1)) == 0) && in dmz_count_bits()
2726 ((end - bit) >= BITS_PER_LONG)) { in dmz_count_bits()
2751 unsigned int nr_blocks = zmd->zone_nr_blocks; in dmz_get_zone_weight()
2764 bitmap = mblk->data; in dmz_get_zone_weight()
2766 nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit); in dmz_get_zone_weight()
2771 nr_blocks -= nr_bits; in dmz_get_zone_weight()
2775 zone->weight = n; in dmz_get_zone_weight()
2788 if (zmd->map_mblk) { in dmz_cleanup_metadata()
2789 for (i = 0; i < zmd->nr_map_blocks; i++) in dmz_cleanup_metadata()
2790 dmz_release_mblock(zmd, zmd->map_mblk[i]); in dmz_cleanup_metadata()
2791 kfree(zmd->map_mblk); in dmz_cleanup_metadata()
2792 zmd->map_mblk = NULL; in dmz_cleanup_metadata()
2795 /* Release super blocks */ in dmz_cleanup_metadata()
2797 if (zmd->sb[i].mblk) { in dmz_cleanup_metadata()
2798 dmz_free_mblock(zmd, zmd->sb[i].mblk); in dmz_cleanup_metadata()
2799 zmd->sb[i].mblk = NULL; in dmz_cleanup_metadata()
2804 while (!list_empty(&zmd->mblk_dirty_list)) { in dmz_cleanup_metadata()
2805 mblk = list_first_entry(&zmd->mblk_dirty_list, in dmz_cleanup_metadata()
2808 (u64)mblk->no, mblk->ref); in dmz_cleanup_metadata()
2809 list_del_init(&mblk->link); in dmz_cleanup_metadata()
2810 rb_erase(&mblk->node, &zmd->mblk_rbtree); in dmz_cleanup_metadata()
2814 while (!list_empty(&zmd->mblk_lru_list)) { in dmz_cleanup_metadata()
2815 mblk = list_first_entry(&zmd->mblk_lru_list, in dmz_cleanup_metadata()
2817 list_del_init(&mblk->link); in dmz_cleanup_metadata()
2818 rb_erase(&mblk->node, &zmd->mblk_rbtree); in dmz_cleanup_metadata()
2823 root = &zmd->mblk_rbtree; in dmz_cleanup_metadata()
2826 (u64)mblk->no, mblk->ref); in dmz_cleanup_metadata()
2827 mblk->ref = 0; in dmz_cleanup_metadata()
2834 mutex_destroy(&zmd->mblk_flush_lock); in dmz_cleanup_metadata()
2835 mutex_destroy(&zmd->map_lock); in dmz_cleanup_metadata()
2840 struct dmz_dev *dev = &zmd->dev[num]; in dmz_print_dev()
2842 if (!bdev_is_zoned(dev->bdev)) in dmz_print_dev()
2845 dmz_dev_info(dev, "Host-managed zoned block device"); in dmz_print_dev()
2847 if (zmd->sb_version > 1) { in dmz_print_dev()
2849 dev->zone_offset << zmd->zone_nr_sectors_shift; in dmz_print_dev()
2851 dmz_dev_info(dev, " %llu 512-byte logical sectors (offset %llu)", in dmz_print_dev()
2852 (u64)dev->capacity, (u64)sector_offset); in dmz_print_dev()
2853 dmz_dev_info(dev, " %u zones of %llu 512-byte logical sectors (offset %llu)", in dmz_print_dev()
2854 dev->nr_zones, (u64)zmd->zone_nr_sectors, in dmz_print_dev()
2855 (u64)dev->zone_offset); in dmz_print_dev()
2857 dmz_dev_info(dev, " %llu 512-byte logical sectors", in dmz_print_dev()
2858 (u64)dev->capacity); in dmz_print_dev()
2859 dmz_dev_info(dev, " %u zones of %llu 512-byte logical sectors", in dmz_print_dev()
2860 dev->nr_zones, (u64)zmd->zone_nr_sectors); in dmz_print_dev()
2878 return -ENOMEM; in dmz_ctr_metadata()
2880 strcpy(zmd->devname, devname); in dmz_ctr_metadata()
2881 zmd->dev = dev; in dmz_ctr_metadata()
2882 zmd->nr_devs = num_dev; in dmz_ctr_metadata()
2883 zmd->mblk_rbtree = RB_ROOT; in dmz_ctr_metadata()
2884 init_rwsem(&zmd->mblk_sem); in dmz_ctr_metadata()
2885 mutex_init(&zmd->mblk_flush_lock); in dmz_ctr_metadata()
2886 spin_lock_init(&zmd->mblk_lock); in dmz_ctr_metadata()
2887 INIT_LIST_HEAD(&zmd->mblk_lru_list); in dmz_ctr_metadata()
2888 INIT_LIST_HEAD(&zmd->mblk_dirty_list); in dmz_ctr_metadata()
2890 mutex_init(&zmd->map_lock); in dmz_ctr_metadata()
2892 atomic_set(&zmd->unmap_nr_cache, 0); in dmz_ctr_metadata()
2893 INIT_LIST_HEAD(&zmd->unmap_cache_list); in dmz_ctr_metadata()
2894 INIT_LIST_HEAD(&zmd->map_cache_list); in dmz_ctr_metadata()
2896 atomic_set(&zmd->nr_reserved_seq_zones, 0); in dmz_ctr_metadata()
2897 INIT_LIST_HEAD(&zmd->reserved_seq_zones_list); in dmz_ctr_metadata()
2899 init_waitqueue_head(&zmd->free_wq); in dmz_ctr_metadata()
2906 /* Get super block */ in dmz_ctr_metadata()
2911 /* Set metadata zones starting from sb_zone */ in dmz_ctr_metadata()
2912 for (i = 0; i < zmd->nr_meta_zones << 1; i++) { in dmz_ctr_metadata()
2913 zone = dmz_get(zmd, zmd->sb[0].zone->id + i); in dmz_ctr_metadata()
2917 ret = -ENXIO; in dmz_ctr_metadata()
2923 ret = -ENXIO; in dmz_ctr_metadata()
2926 set_bit(DMZ_META, &zone->flags); in dmz_ctr_metadata()
2934 * Cache size boundaries: allow at least 2 super blocks, the chunk map in dmz_ctr_metadata()
2939 zmd->min_nr_mblks = 2 + zmd->nr_map_blocks + zmd->zone_nr_bitmap_blocks * 16; in dmz_ctr_metadata()
2940 zmd->max_nr_mblks = zmd->min_nr_mblks + 512; in dmz_ctr_metadata()
2943 zmd->mblk_shrinker = shrinker_alloc(0, "dm-zoned-meta:(%u:%u)", in dmz_ctr_metadata()
2944 MAJOR(dev->bdev->bd_dev), in dmz_ctr_metadata()
2945 MINOR(dev->bdev->bd_dev)); in dmz_ctr_metadata()
2946 if (!zmd->mblk_shrinker) { in dmz_ctr_metadata()
2947 ret = -ENOMEM; in dmz_ctr_metadata()
2952 zmd->mblk_shrinker->count_objects = dmz_mblock_shrinker_count; in dmz_ctr_metadata()
2953 zmd->mblk_shrinker->scan_objects = dmz_mblock_shrinker_scan; in dmz_ctr_metadata()
2954 zmd->mblk_shrinker->private_data = zmd; in dmz_ctr_metadata()
2956 shrinker_register(zmd->mblk_shrinker); in dmz_ctr_metadata()
2958 dmz_zmd_info(zmd, "DM-Zoned metadata version %d", zmd->sb_version); in dmz_ctr_metadata()
2959 for (i = 0; i < zmd->nr_devs; i++) in dmz_ctr_metadata()
2962 dmz_zmd_info(zmd, " %u zones of %llu 512-byte logical sectors", in dmz_ctr_metadata()
2963 zmd->nr_zones, (u64)zmd->zone_nr_sectors); in dmz_ctr_metadata()
2965 zmd->nr_meta_zones * 2); in dmz_ctr_metadata()
2967 zmd->nr_data_zones, zmd->nr_chunks); in dmz_ctr_metadata()
2969 zmd->nr_cache, atomic_read(&zmd->unmap_nr_cache)); in dmz_ctr_metadata()
2970 for (i = 0; i < zmd->nr_devs; i++) { in dmz_ctr_metadata()
2979 zmd->nr_reserved_seq); in dmz_ctr_metadata()
2981 dmz_zmd_debug(zmd, "%u metadata blocks per set (%u max cache)", in dmz_ctr_metadata()
2982 zmd->nr_meta_blocks, zmd->max_nr_mblks); in dmz_ctr_metadata()
2984 zmd->nr_map_blocks); in dmz_ctr_metadata()
2986 zmd->nr_bitmap_blocks); in dmz_ctr_metadata()
3004 shrinker_free(zmd->mblk_shrinker); in dmz_dtr_metadata()
3020 for (i = 0; i < zmd->nr_zones; i++) { in dmz_resume_metadata()
3024 return -EIO; in dmz_resume_metadata()
3026 wp_block = zone->wp_block; in dmz_resume_metadata()
3041 zone->wp_block = 0; in dmz_resume_metadata()
3042 else if (zone->wp_block != wp_block) { in dmz_resume_metadata()
3044 i, (u64)zone->wp_block, (u64)wp_block); in dmz_resume_metadata()
3045 zone->wp_block = wp_block; in dmz_resume_metadata()
3046 dmz_invalidate_blocks(zmd, zone, zone->wp_block, in dmz_resume_metadata()
3047 zmd->zone_nr_blocks - zone->wp_block); in dmz_resume_metadata()