273 static inline int superblock_read_lock(struct dm_clone_metadata *cmd,  in superblock_read_lock()  argument
276 	return dm_bm_read_lock(cmd->bm, SUPERBLOCK_LOCATION, &sb_validator, sblock);  in superblock_read_lock()
279 static inline int superblock_write_lock_zero(struct dm_clone_metadata *cmd,  in superblock_write_lock_zero()  argument
282 	return dm_bm_write_lock_zero(cmd->bm, SUPERBLOCK_LOCATION, &sb_validator, sblock);  in superblock_write_lock_zero()
285 static int __copy_sm_root(struct dm_clone_metadata *cmd)  in __copy_sm_root()  argument
290 	r = dm_sm_root_size(cmd->sm, &root_size);  in __copy_sm_root()
294 	return dm_sm_copy_root(cmd->sm, &cmd->metadata_space_map_root, root_size);  in __copy_sm_root()
298 static void __prepare_superblock(struct dm_clone_metadata *cmd,  in __prepare_superblock()  argument
310 	memcpy(&sb->metadata_space_map_root, &cmd->metadata_space_map_root,  in __prepare_superblock()
311 	       sizeof(cmd->metadata_space_map_root));  in __prepare_superblock()
313 	sb->region_size = cpu_to_le64(cmd->region_size);  in __prepare_superblock()
314 	sb->target_size = cpu_to_le64(cmd->target_size);  in __prepare_superblock()
315 	sb->bitset_root = cpu_to_le64(cmd->bitset_root);  in __prepare_superblock()
318 static int __open_metadata(struct dm_clone_metadata *cmd)  in __open_metadata()  argument
324 	r = superblock_read_lock(cmd, &sblock);  in __open_metadata()
334 	if (cmd->region_size != le64_to_cpu(sb->region_size) ||  in __open_metadata()
335 	    cmd->target_size != le64_to_cpu(sb->target_size)) {  in __open_metadata()
341 	r = dm_tm_open_with_sm(cmd->bm, SUPERBLOCK_LOCATION,  in __open_metadata()
344 			       &cmd->tm, &cmd->sm);  in __open_metadata()
351 	dm_disk_bitset_init(cmd->tm, &cmd->bitset_info);  in __open_metadata()
352 	cmd->bitset_root = le64_to_cpu(sb->bitset_root);  in __open_metadata()
360 static int __format_metadata(struct dm_clone_metadata *cmd)  in __format_metadata()  argument
366 	r = dm_tm_create_with_sm(cmd->bm, SUPERBLOCK_LOCATION, &cmd->tm, &cmd->sm);  in __format_metadata()
372 	dm_disk_bitset_init(cmd->tm, &cmd->bitset_info);  in __format_metadata()
374 	r = dm_bitset_empty(&cmd->bitset_info, &cmd->bitset_root);  in __format_metadata()
380 	r = dm_bitset_resize(&cmd->bitset_info, cmd->bitset_root, 0,  in __format_metadata()
381 			     cmd->nr_regions, false, &cmd->bitset_root);  in __format_metadata()
383 		DMERR("Failed to resize on-disk bitset to %lu entries", cmd->nr_regions);  in __format_metadata()
388 	r = dm_tm_pre_commit(cmd->tm);  in __format_metadata()
394 	r = __copy_sm_root(cmd);  in __format_metadata()
400 	r = superblock_write_lock_zero(cmd, &sblock);  in __format_metadata()
407 	__prepare_superblock(cmd, sb);  in __format_metadata()
408 	r = dm_tm_commit(cmd->tm, sblock);  in __format_metadata()
417 	dm_sm_destroy(cmd->sm);  in __format_metadata()
418 	dm_tm_destroy(cmd->tm);  in __format_metadata()
423 static int __open_or_format_metadata(struct dm_clone_metadata *cmd, bool may_format_device)  in __open_or_format_metadata()  argument
428 	r = __superblock_all_zeroes(cmd->bm, &formatted);  in __open_or_format_metadata()
433 		return may_format_device ? __format_metadata(cmd) : -EPERM;  in __open_or_format_metadata()
435 	return __open_metadata(cmd);  in __open_or_format_metadata()
438 static int __create_persistent_data_structures(struct dm_clone_metadata *cmd,  in __create_persistent_data_structures()  argument
444 	cmd->bm = dm_block_manager_create(cmd->bdev,  in __create_persistent_data_structures()
447 	if (IS_ERR(cmd->bm)) {  in __create_persistent_data_structures()
449 		return PTR_ERR(cmd->bm);  in __create_persistent_data_structures()
452 	r = __open_or_format_metadata(cmd, may_format_device);  in __create_persistent_data_structures()
454 		dm_block_manager_destroy(cmd->bm);  in __create_persistent_data_structures()
459 static void __destroy_persistent_data_structures(struct dm_clone_metadata *cmd)  in __destroy_persistent_data_structures()  argument
461 	dm_sm_destroy(cmd->sm);  in __destroy_persistent_data_structures()
462 	dm_tm_destroy(cmd->tm);  in __destroy_persistent_data_structures()
463 	dm_block_manager_destroy(cmd->bm);  in __destroy_persistent_data_structures()
492 static int dirty_map_init(struct dm_clone_metadata *cmd)  in dirty_map_init()  argument
494 	if (__dirty_map_init(&cmd->dmap[0], cmd->nr_words, cmd->nr_regions)) {  in dirty_map_init()
499 	if (__dirty_map_init(&cmd->dmap[1], cmd->nr_words, cmd->nr_regions)) {  in dirty_map_init()
501 		__dirty_map_exit(&cmd->dmap[0]);  in dirty_map_init()
505 	cmd->current_dmap = &cmd->dmap[0];  in dirty_map_init()
506 	cmd->committing_dmap = NULL;  in dirty_map_init()
511 static void dirty_map_exit(struct dm_clone_metadata *cmd)  in dirty_map_exit()  argument
513 	__dirty_map_exit(&cmd->dmap[0]);  in dirty_map_exit()
514 	__dirty_map_exit(&cmd->dmap[1]);  in dirty_map_exit()
517 static int __load_bitset_in_core(struct dm_clone_metadata *cmd)  in __load_bitset_in_core()  argument
524 	r = dm_bitset_flush(&cmd->bitset_info, cmd->bitset_root, &cmd->bitset_root);  in __load_bitset_in_core()
528 	r = dm_bitset_cursor_begin(&cmd->bitset_info, cmd->bitset_root, cmd->nr_regions, &c);  in __load_bitset_in_core()
533 		__assign_bit(i, cmd->region_map, dm_bitset_cursor_get_value(&c));  in __load_bitset_in_core()
535 		if (i >= (cmd->nr_regions - 1))  in __load_bitset_in_core()
554 	struct dm_clone_metadata *cmd;  in dm_clone_metadata_open()  local
556 	cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);  in dm_clone_metadata_open()
557 	if (!cmd) {  in dm_clone_metadata_open()
562 	cmd->bdev = bdev;  in dm_clone_metadata_open()
563 	cmd->target_size = target_size;  in dm_clone_metadata_open()
564 	cmd->region_size = region_size;  in dm_clone_metadata_open()
565 	cmd->nr_regions = dm_sector_div_up(cmd->target_size, cmd->region_size);  in dm_clone_metadata_open()
566 	cmd->nr_words = BITS_TO_LONGS(cmd->nr_regions);  in dm_clone_metadata_open()
568 	init_rwsem(&cmd->lock);  in dm_clone_metadata_open()
569 	spin_lock_init(&cmd->bitmap_lock);  in dm_clone_metadata_open()
570 	cmd->read_only = 0;  in dm_clone_metadata_open()
571 	cmd->fail_io = false;  in dm_clone_metadata_open()
572 	cmd->hydration_done = false;  in dm_clone_metadata_open()
574 	cmd->region_map = kvmalloc(bitmap_size(cmd->nr_regions), GFP_KERNEL);  in dm_clone_metadata_open()
575 	if (!cmd->region_map) {  in dm_clone_metadata_open()
581 	r = __create_persistent_data_structures(cmd, true);  in dm_clone_metadata_open()
585 	r = __load_bitset_in_core(cmd);  in dm_clone_metadata_open()
591 	r = dirty_map_init(cmd);  in dm_clone_metadata_open()
595 	if (bitmap_full(cmd->region_map, cmd->nr_regions))  in dm_clone_metadata_open()
596 		cmd->hydration_done = true;  in dm_clone_metadata_open()
598 	return cmd;  in dm_clone_metadata_open()
601 	__destroy_persistent_data_structures(cmd);  in dm_clone_metadata_open()
604 	kvfree(cmd->region_map);  in dm_clone_metadata_open()
607 	kfree(cmd);  in dm_clone_metadata_open()
612 void dm_clone_metadata_close(struct dm_clone_metadata *cmd)  in dm_clone_metadata_close()  argument
614 	if (!cmd->fail_io)  in dm_clone_metadata_close()
615 		__destroy_persistent_data_structures(cmd);  in dm_clone_metadata_close()
617 	dirty_map_exit(cmd);  in dm_clone_metadata_close()
618 	kvfree(cmd->region_map);  in dm_clone_metadata_close()
619 	kfree(cmd);  in dm_clone_metadata_close()
622 bool dm_clone_is_hydration_done(struct dm_clone_metadata *cmd)  in dm_clone_is_hydration_done()  argument
624 	return cmd->hydration_done;  in dm_clone_is_hydration_done()
627 bool dm_clone_is_region_hydrated(struct dm_clone_metadata *cmd, unsigned long region_nr)  in dm_clone_is_region_hydrated()  argument
629 	return dm_clone_is_hydration_done(cmd) || test_bit(region_nr, cmd->region_map);  in dm_clone_is_region_hydrated()
632 bool dm_clone_is_range_hydrated(struct dm_clone_metadata *cmd,  in dm_clone_is_range_hydrated()  argument
637 	if (dm_clone_is_hydration_done(cmd))  in dm_clone_is_range_hydrated()
640 	bit = find_next_zero_bit(cmd->region_map, cmd->nr_regions, start);  in dm_clone_is_range_hydrated()
645 unsigned int dm_clone_nr_of_hydrated_regions(struct dm_clone_metadata *cmd)  in dm_clone_nr_of_hydrated_regions()  argument
647 	return bitmap_weight(cmd->region_map, cmd->nr_regions);  in dm_clone_nr_of_hydrated_regions()
650 unsigned long dm_clone_find_next_unhydrated_region(struct dm_clone_metadata *cmd,  in dm_clone_find_next_unhydrated_region()  argument
653 	return find_next_zero_bit(cmd->region_map, cmd->nr_regions, start);  in dm_clone_find_next_unhydrated_region()
656 static int __update_metadata_word(struct dm_clone_metadata *cmd,  in __update_metadata_word()  argument
662 	unsigned long max_index = min(cmd->nr_regions, (word + 1) * BITS_PER_LONG);  in __update_metadata_word()
666 			r = dm_bitset_set_bit(&cmd->bitset_info, cmd->bitset_root,  in __update_metadata_word()
667 					      index, &cmd->bitset_root);  in __update_metadata_word()
680 static int __metadata_commit(struct dm_clone_metadata *cmd)  in __metadata_commit()  argument
687 	r = dm_bitset_flush(&cmd->bitset_info, cmd->bitset_root, &cmd->bitset_root);  in __metadata_commit()
694 	r = dm_tm_pre_commit(cmd->tm);  in __metadata_commit()
700 	/* Save the space map root in cmd->metadata_space_map_root */  in __metadata_commit()
701 	r = __copy_sm_root(cmd);  in __metadata_commit()
708 	r = superblock_write_lock_zero(cmd, &sblock);  in __metadata_commit()
716 	__prepare_superblock(cmd, sb);  in __metadata_commit()
719 	r = dm_tm_commit(cmd->tm, sblock);  in __metadata_commit()
728 	if (bitmap_full(cmd->region_map, cmd->nr_regions))  in __metadata_commit()
729 		cmd->hydration_done = true;  in __metadata_commit()
734 static int __flush_dmap(struct dm_clone_metadata *cmd, struct dirty_map *dmap)  in __flush_dmap()  argument
741 		word = find_next_bit(dmap->dirty_words, cmd->nr_words, word);  in __flush_dmap()
743 		if (word == cmd->nr_words)  in __flush_dmap()
746 		r = __update_metadata_word(cmd, dmap->dirty_regions, word);  in __flush_dmap()
753 	} while (word < cmd->nr_words);  in __flush_dmap()
755 	r = __metadata_commit(cmd);  in __flush_dmap()
761 	spin_lock_irq(&cmd->bitmap_lock);  in __flush_dmap()
763 	spin_unlock_irq(&cmd->bitmap_lock);  in __flush_dmap()
768 int dm_clone_metadata_pre_commit(struct dm_clone_metadata *cmd)  in dm_clone_metadata_pre_commit()  argument
773 	down_write(&cmd->lock);  in dm_clone_metadata_pre_commit()
775 	if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) {  in dm_clone_metadata_pre_commit()
781 	dmap = cmd->current_dmap;  in dm_clone_metadata_pre_commit()
784 	next_dmap = (dmap == &cmd->dmap[0]) ? &cmd->dmap[1] : &cmd->dmap[0];  in dm_clone_metadata_pre_commit()
790 	if (WARN_ON(next_dmap->changed || cmd->committing_dmap)) {  in dm_clone_metadata_pre_commit()
796 	spin_lock_irq(&cmd->bitmap_lock);  in dm_clone_metadata_pre_commit()
797 	cmd->current_dmap = next_dmap;  in dm_clone_metadata_pre_commit()
798 	spin_unlock_irq(&cmd->bitmap_lock);  in dm_clone_metadata_pre_commit()
801 	cmd->committing_dmap = dmap;  in dm_clone_metadata_pre_commit()
803 	up_write(&cmd->lock);  in dm_clone_metadata_pre_commit()
808 int dm_clone_metadata_commit(struct dm_clone_metadata *cmd)  in dm_clone_metadata_commit()  argument
812 	down_write(&cmd->lock);  in dm_clone_metadata_commit()
814 	if (cmd->fail_io || dm_bm_is_read_only(cmd->bm))  in dm_clone_metadata_commit()
817 	if (WARN_ON(!cmd->committing_dmap)) {  in dm_clone_metadata_commit()
822 	r = __flush_dmap(cmd, cmd->committing_dmap);  in dm_clone_metadata_commit()
825 		cmd->committing_dmap = NULL;  in dm_clone_metadata_commit()
828 	up_write(&cmd->lock);  in dm_clone_metadata_commit()
833 int dm_clone_set_region_hydrated(struct dm_clone_metadata *cmd, unsigned long region_nr)  in dm_clone_set_region_hydrated()  argument
839 	if (unlikely(region_nr >= cmd->nr_regions)) {  in dm_clone_set_region_hydrated()
841 		      region_nr, cmd->nr_regions);  in dm_clone_set_region_hydrated()
847 	spin_lock_irqsave(&cmd->bitmap_lock, flags);  in dm_clone_set_region_hydrated()
849 	if (cmd->read_only) {  in dm_clone_set_region_hydrated()
854 	dmap = cmd->current_dmap;  in dm_clone_set_region_hydrated()
858 	__set_bit(region_nr, cmd->region_map);  in dm_clone_set_region_hydrated()
862 	spin_unlock_irqrestore(&cmd->bitmap_lock, flags);  in dm_clone_set_region_hydrated()
867 int dm_clone_cond_set_range(struct dm_clone_metadata *cmd, unsigned long start,  in dm_clone_cond_set_range()  argument
874 	if (unlikely(start >= cmd->nr_regions || (start + nr_regions) < start ||  in dm_clone_cond_set_range()
875 		     (start + nr_regions) > cmd->nr_regions)) {  in dm_clone_cond_set_range()
877 		      start, nr_regions, cmd->nr_regions);  in dm_clone_cond_set_range()
881 	spin_lock_irq(&cmd->bitmap_lock);  in dm_clone_cond_set_range()
883 	if (cmd->read_only) {  in dm_clone_cond_set_range()
888 	dmap = cmd->current_dmap;  in dm_clone_cond_set_range()
890 		if (!test_bit(region_nr, cmd->region_map)) {  in dm_clone_cond_set_range()
894 			__set_bit(region_nr, cmd->region_map);  in dm_clone_cond_set_range()
899 	spin_unlock_irq(&cmd->bitmap_lock);  in dm_clone_cond_set_range()
907  * cmd->region_map without taking the cmd->bitmap_lock spinlock. The only
914 int dm_clone_reload_in_core_bitset(struct dm_clone_metadata *cmd)  in dm_clone_reload_in_core_bitset()  argument
918 	down_write(&cmd->lock);  in dm_clone_reload_in_core_bitset()
920 	if (cmd->fail_io)  in dm_clone_reload_in_core_bitset()
923 	r = __load_bitset_in_core(cmd);  in dm_clone_reload_in_core_bitset()
925 	up_write(&cmd->lock);  in dm_clone_reload_in_core_bitset()
930 bool dm_clone_changed_this_transaction(struct dm_clone_metadata *cmd)  in dm_clone_changed_this_transaction()  argument
935 	spin_lock_irqsave(&cmd->bitmap_lock, flags);  in dm_clone_changed_this_transaction()
936 	r = cmd->dmap[0].changed || cmd->dmap[1].changed;  in dm_clone_changed_this_transaction()
937 	spin_unlock_irqrestore(&cmd->bitmap_lock, flags);  in dm_clone_changed_this_transaction()
942 int dm_clone_metadata_abort(struct dm_clone_metadata *cmd)  in dm_clone_metadata_abort()  argument
946 	down_write(&cmd->lock);  in dm_clone_metadata_abort()
948 	if (cmd->fail_io || dm_bm_is_read_only(cmd->bm))  in dm_clone_metadata_abort()
951 	__destroy_persistent_data_structures(cmd);  in dm_clone_metadata_abort()
953 	r = __create_persistent_data_structures(cmd, false);  in dm_clone_metadata_abort()
956 		cmd->fail_io = true;  in dm_clone_metadata_abort()
959 	up_write(&cmd->lock);  in dm_clone_metadata_abort()
964 void dm_clone_metadata_set_read_only(struct dm_clone_metadata *cmd)  in dm_clone_metadata_set_read_only()  argument
966 	down_write(&cmd->lock);  in dm_clone_metadata_set_read_only()
968 	spin_lock_irq(&cmd->bitmap_lock);  in dm_clone_metadata_set_read_only()
969 	cmd->read_only = 1;  in dm_clone_metadata_set_read_only()
970 	spin_unlock_irq(&cmd->bitmap_lock);  in dm_clone_metadata_set_read_only()
972 	if (!cmd->fail_io)  in dm_clone_metadata_set_read_only()
973 		dm_bm_set_read_only(cmd->bm);  in dm_clone_metadata_set_read_only()
975 	up_write(&cmd->lock);  in dm_clone_metadata_set_read_only()
978 void dm_clone_metadata_set_read_write(struct dm_clone_metadata *cmd)  in dm_clone_metadata_set_read_write()  argument
980 	down_write(&cmd->lock);  in dm_clone_metadata_set_read_write()
982 	spin_lock_irq(&cmd->bitmap_lock);  in dm_clone_metadata_set_read_write()
983 	cmd->read_only = 0;  in dm_clone_metadata_set_read_write()
984 	spin_unlock_irq(&cmd->bitmap_lock);  in dm_clone_metadata_set_read_write()
986 	if (!cmd->fail_io)  in dm_clone_metadata_set_read_write()
987 		dm_bm_set_read_write(cmd->bm);  in dm_clone_metadata_set_read_write()
989 	up_write(&cmd->lock);  in dm_clone_metadata_set_read_write()
992 int dm_clone_get_free_metadata_block_count(struct dm_clone_metadata *cmd,  in dm_clone_get_free_metadata_block_count()  argument
997 	down_read(&cmd->lock);  in dm_clone_get_free_metadata_block_count()
999 	if (!cmd->fail_io)  in dm_clone_get_free_metadata_block_count()
1000 		r = dm_sm_get_nr_free(cmd->sm, result);  in dm_clone_get_free_metadata_block_count()
1002 	up_read(&cmd->lock);  in dm_clone_get_free_metadata_block_count()
1007 int dm_clone_get_metadata_dev_size(struct dm_clone_metadata *cmd,  in dm_clone_get_metadata_dev_size()  argument
1012 	down_read(&cmd->lock);  in dm_clone_get_metadata_dev_size()
1014 	if (!cmd->fail_io)  in dm_clone_get_metadata_dev_size()
1015 		r = dm_sm_get_nr_blocks(cmd->sm, result);  in dm_clone_get_metadata_dev_size()
1017 	up_read(&cmd->lock);  in dm_clone_get_metadata_dev_size()