55 	struct uds_index *index;  member
80 	return uds_is_chapter_sparse(zone->index->volume->geometry,  in is_zone_chapter_sparse()
86 			       struct uds_index *index)  in launch_zone_message()  argument
95 	request->index = index;  in launch_zone_message()
104 static void enqueue_barrier_messages(struct uds_index *index, u64 virtual_chapter)  in enqueue_barrier_messages()  argument
112 	for (zone = 0; zone < index->zone_count; zone++) {  in enqueue_barrier_messages()
113 		int result = launch_zone_message(message, zone, index);  in enqueue_barrier_messages()
124 static u64 triage_index_request(struct uds_index *index, struct uds_request *request)  in triage_index_request()  argument
129 	virtual_chapter = uds_lookup_volume_index_name(index->volume_index,  in triage_index_request()
134 	zone = index->zones[request->zone_number];  in triage_index_request()
156 	if ((zone->index->zone_count > 1) ||  in simulate_index_zone_barrier_message()
157 	    !uds_is_sparse_index_geometry(zone->index->volume->geometry))  in simulate_index_zone_barrier_message()
160 	sparse_virtual_chapter = triage_index_request(zone->index, request);  in simulate_index_zone_barrier_message()
170 	struct uds_index *index = request->index;  in triage_request()  local
171 	u64 sparse_virtual_chapter = triage_index_request(index, request);  in triage_request()
174 		enqueue_barrier_messages(index, sparse_virtual_chapter);  in triage_request()
179 static int finish_previous_chapter(struct uds_index *index, u64 current_chapter_number)  in finish_previous_chapter()  argument
182 	struct chapter_writer *writer = index->chapter_writer;  in finish_previous_chapter()
185 	while (index->newest_virtual_chapter < current_chapter_number)  in finish_previous_chapter()
201 	result = finish_previous_chapter(zone->index, zone->newest_virtual_chapter);  in swap_open_chapter()
213 static unsigned int start_closing_chapter(struct uds_index *index,  in start_closing_chapter()  argument
218 	struct chapter_writer *writer = index->chapter_writer;  in start_closing_chapter()
238 	for (i = 0; i < zone->index->zone_count; i++) {  in announce_chapter_closed()
242 		result = launch_zone_message(zone_message, i, zone->index);  in announce_chapter_closed()
268 	uds_set_volume_index_zone_open_chapter(zone->index->volume_index, zone->id,  in open_next_chapter()
272 	finished_zones = start_closing_chapter(zone->index, zone->id,  in open_next_chapter()
274 	if ((finished_zones == 1) && (zone->index->zone_count > 1)) {  in open_next_chapter()
281 	expire_chapters = uds_chapters_to_expire(zone->index->volume->geometry,  in open_next_chapter()
285 	if (finished_zones < zone->index->zone_count)  in open_next_chapter()
289 		uds_forget_chapter(zone->index->volume, expiring++);  in open_next_chapter()
305 	struct index_zone *zone = request->index->zones[request->zone_number];  in dispatch_index_zone_control_request()
355 	volume = zone->index->volume;  in search_sparse_cache_in_zone()
388 	volume = zone->index->volume;  in get_record_from_zone()
419 	result = uds_get_volume_index_record(zone->index->volume_index,  in search_index_zone()
473 		} else if (uds_is_sparse_index_geometry(zone->index->volume->geometry) &&  in search_index_zone()
474 			   !uds_is_volume_index_sample(zone->index->volume_index,  in search_index_zone()
525 	result = uds_get_volume_index_record(zone->index->volume_index,  in remove_from_index_zone()
576 static int dispatch_index_request(struct uds_index *index, struct uds_request *request)  in dispatch_index_request()  argument
579 	struct index_zone *zone = index->zones[request->zone_number];  in dispatch_index_request()
613 	struct uds_index *index = request->index;  in execute_zone_request()  local
627 	index->need_to_save = true;  in execute_zone_request()
630 		index->callback(request);  in execute_zone_request()
634 	result = dispatch_index_request(index, request);  in execute_zone_request()
644 	index->callback(request);  in execute_zone_request()
647 static int initialize_index_queues(struct uds_index *index,  in initialize_index_queues()  argument
653 	for (i = 0; i < index->zone_count; i++) {  in initialize_index_queues()
655 						&index->zone_queues[i]);  in initialize_index_queues()
661 	if ((index->zone_count > 1) && uds_is_sparse_index_geometry(geometry)) {  in initialize_index_queues()
663 						&index->triage_queue);  in initialize_index_queues()
676 	struct uds_index *index = writer->index;  in close_chapters()  local
681 		while (writer->zones_to_write < index->zone_count) {  in close_chapters()
701 		if (index->has_saved_open_chapter) {  in close_chapters()
708 			index->has_saved_open_chapter = false;  in close_chapters()
709 			result = uds_discard_open_chapter(index->layout);  in close_chapters()
714 		result = uds_close_open_chapter(writer->chapters, index->zone_count,  in close_chapters()
715 						index->volume,  in close_chapters()
718 						index->newest_virtual_chapter);  in close_chapters()
721 		index->newest_virtual_chapter++;  in close_chapters()
722 		index->oldest_virtual_chapter +=  in close_chapters()
723 			uds_chapters_to_expire(index->volume->geometry,  in close_chapters()
724 					       index->newest_virtual_chapter);  in close_chapters()
759 static int make_chapter_writer(struct uds_index *index,  in make_chapter_writer()  argument
765 		(sizeof(struct uds_volume_record) * index->volume->geometry->records_per_chapter);  in make_chapter_writer()
767 	result = vdo_allocate_extended(struct chapter_writer, index->zone_count,  in make_chapter_writer()
773 	writer->index = index;  in make_chapter_writer()
785 					     index->volume->geometry,  in make_chapter_writer()
786 					     index->volume->nonce);  in make_chapter_writer()
793 			       index->zone_count * sizeof(struct open_chapter_zone *) +  in make_chapter_writer()
807 static int load_index(struct uds_index *index)  in load_index()  argument
812 	result = uds_load_index_state(index->layout, index);  in load_index()
816 	last_save_chapter = ((index->last_save != NO_LAST_SAVE) ? index->last_save : 0);  in load_index()
819 		     (unsigned long long) index->oldest_virtual_chapter,  in load_index()
825 static int rebuild_index_page_map(struct uds_index *index, u64 vcn)  in rebuild_index_page_map()  argument
829 	struct index_geometry *geometry = index->volume->geometry;  in rebuild_index_page_map()
839 		result = uds_get_volume_index_page(index->volume, chapter,  in rebuild_index_page_map()
856 		uds_update_index_page_map(index->volume->index_page_map, vcn, chapter,  in rebuild_index_page_map()
864 static int replay_record(struct uds_index *index, const struct uds_record_name *name,  in replay_record()  argument
872 	    !uds_is_volume_index_sample(index->volume_index, name)) {  in replay_record()
880 	result = uds_get_volume_index_record(index->volume_index, name, &record);  in replay_record()
910 			result = uds_search_volume_page_cache_for_rebuild(index->volume,  in replay_record()
945 static bool check_for_suspend(struct uds_index *index)  in check_for_suspend()  argument
949 	if (index->load_context == NULL)  in check_for_suspend()
952 	mutex_lock(&index->load_context->mutex);  in check_for_suspend()
953 	if (index->load_context->status != INDEX_SUSPENDING) {  in check_for_suspend()
954 		mutex_unlock(&index->load_context->mutex);  in check_for_suspend()
959 	index->load_context->status = INDEX_SUSPENDED;  in check_for_suspend()
960 	uds_broadcast_cond(&index->load_context->cond);  in check_for_suspend()
962 	while ((index->load_context->status != INDEX_OPENING) &&  in check_for_suspend()
963 	       (index->load_context->status != INDEX_FREEING))  in check_for_suspend()
964 		uds_wait_cond(&index->load_context->cond, &index->load_context->mutex);  in check_for_suspend()
966 	closing = (index->load_context->status == INDEX_FREEING);  in check_for_suspend()
967 	mutex_unlock(&index->load_context->mutex);  in check_for_suspend()
971 static int replay_chapter(struct uds_index *index, u64 virtual, bool sparse)  in replay_chapter()  argument
979 	if (check_for_suspend(index)) {  in replay_chapter()
985 	geometry = index->volume->geometry;  in replay_chapter()
987 	uds_prefetch_volume_chapter(index->volume, physical_chapter);  in replay_chapter()
988 	uds_set_volume_index_open_chapter(index->volume_index, virtual);  in replay_chapter()
990 	result = rebuild_index_page_map(index, virtual);  in replay_chapter()
1002 		result = uds_get_volume_record_page(index->volume, physical_chapter,  in replay_chapter()
1015 			result = replay_record(index, &name, virtual, sparse);  in replay_chapter()
1024 static int replay_volume(struct uds_index *index)  in replay_volume()  argument
1030 	u64 from_virtual = index->oldest_virtual_chapter;  in replay_volume()
1031 	u64 upto_virtual = index->newest_virtual_chapter;  in replay_volume()
1049 	old_map_update = index->volume->index_page_map->last_update;  in replay_volume()
1051 		will_be_sparse = uds_is_chapter_sparse(index->volume->geometry,  in replay_volume()
1054 		result = replay_chapter(index, virtual, will_be_sparse);  in replay_volume()
1060 	uds_set_volume_index_open_chapter(index->volume_index, upto_virtual);  in replay_volume()
1062 	new_map_update = index->volume->index_page_map->last_update;  in replay_volume()
1072 static int rebuild_index(struct uds_index *index)  in rebuild_index()  argument
1078 	u32 chapters_per_volume = index->volume->geometry->chapters_per_volume;  in rebuild_index()
1080 	index->volume->lookup_mode = LOOKUP_FOR_REBUILD;  in rebuild_index()
1081 	result = uds_find_volume_chapter_boundaries(index->volume, &lowest, &highest,  in rebuild_index()
1089 		index->newest_virtual_chapter = 0;  in rebuild_index()
1090 		index->oldest_virtual_chapter = 0;  in rebuild_index()
1091 		index->volume->lookup_mode = LOOKUP_NORMAL;  in rebuild_index()
1095 	index->newest_virtual_chapter = highest + 1;  in rebuild_index()
1096 	index->oldest_virtual_chapter = lowest;  in rebuild_index()
1097 	if (index->newest_virtual_chapter ==  in rebuild_index()
1098 	    (index->oldest_virtual_chapter + chapters_per_volume)) {  in rebuild_index()
1100 		index->oldest_virtual_chapter++;  in rebuild_index()
1103 	result = replay_volume(index);  in rebuild_index()
1107 	index->volume->lookup_mode = LOOKUP_NORMAL;  in rebuild_index()
1121 static int make_index_zone(struct uds_index *index, unsigned int zone_number)  in make_index_zone()  argument
1130 	result = uds_make_open_chapter(index->volume->geometry, index->zone_count,  in make_index_zone()
1137 	result = uds_make_open_chapter(index->volume->geometry, index->zone_count,  in make_index_zone()
1144 	zone->index = index;  in make_index_zone()
1146 	index->zones[zone_number] = zone;  in make_index_zone()
1158 	struct uds_index *index = NULL;  in uds_make_index()  local
1164 				       struct uds_request_queue *, "index", &index);  in uds_make_index()
1168 	index->zone_count = config->zone_count;  in uds_make_index()
1170 	result = uds_make_index_layout(config, new, &index->layout);  in uds_make_index()
1172 		uds_free_index(index);  in uds_make_index()
1176 	result = vdo_allocate(index->zone_count, struct index_zone *, "zones",  in uds_make_index()
1177 			      &index->zones);  in uds_make_index()
1179 		uds_free_index(index);  in uds_make_index()
1183 	result = uds_make_volume(config, index->layout, &index->volume);  in uds_make_index()
1185 		uds_free_index(index);  in uds_make_index()
1189 	index->volume->lookup_mode = LOOKUP_NORMAL;  in uds_make_index()
1190 	for (z = 0; z < index->zone_count; z++) {  in uds_make_index()
1191 		result = make_index_zone(index, z);  in uds_make_index()
1193 			uds_free_index(index);  in uds_make_index()
1199 	nonce = uds_get_volume_nonce(index->layout);  in uds_make_index()
1200 	result = uds_make_volume_index(config, nonce, &index->volume_index);  in uds_make_index()
1202 		uds_free_index(index);  in uds_make_index()
1206 	index->load_context = load_context;  in uds_make_index()
1207 	index->callback = callback;  in uds_make_index()
1209 	result = initialize_index_queues(index, config->geometry);  in uds_make_index()
1211 		uds_free_index(index);  in uds_make_index()
1215 	result = make_chapter_writer(index, &index->chapter_writer);  in uds_make_index()
1217 		uds_free_index(index);  in uds_make_index()
1222 		result = load_index(index);  in uds_make_index()
1234 				result = rebuild_index(index);  in uds_make_index()
1245 		uds_free_index(index);  in uds_make_index()
1249 	for (z = 0; z < index->zone_count; z++) {  in uds_make_index()
1250 		zone = index->zones[z];  in uds_make_index()
1251 		zone->oldest_virtual_chapter = index->oldest_virtual_chapter;  in uds_make_index()
1252 		zone->newest_virtual_chapter = index->newest_virtual_chapter;  in uds_make_index()
1255 	if (index->load_context != NULL) {  in uds_make_index()
1256 		mutex_lock(&index->load_context->mutex);  in uds_make_index()
1257 		index->load_context->status = INDEX_READY;  in uds_make_index()
1262 		uds_broadcast_cond(&index->load_context->cond);  in uds_make_index()
1263 		mutex_unlock(&index->load_context->mutex);  in uds_make_index()
1266 	index->has_saved_open_chapter = loaded;  in uds_make_index()
1267 	index->need_to_save = !loaded;  in uds_make_index()
1268 	*new_index = index;  in uds_make_index()
1272 void uds_free_index(struct uds_index *index)  in uds_free_index()  argument
1276 	if (index == NULL)  in uds_free_index()
1279 	uds_request_queue_finish(index->triage_queue);  in uds_free_index()
1280 	for (i = 0; i < index->zone_count; i++)  in uds_free_index()
1281 		uds_request_queue_finish(index->zone_queues[i]);  in uds_free_index()
1283 	free_chapter_writer(index->chapter_writer);  in uds_free_index()
1285 	uds_free_volume_index(index->volume_index);  in uds_free_index()
1286 	if (index->zones != NULL) {  in uds_free_index()
1287 		for (i = 0; i < index->zone_count; i++)  in uds_free_index()
1288 			free_index_zone(index->zones[i]);  in uds_free_index()
1289 		vdo_free(index->zones);  in uds_free_index()
1292 	uds_free_volume(index->volume);  in uds_free_index()
1293 	uds_free_index_layout(vdo_forget(index->layout));  in uds_free_index()
1294 	vdo_free(index);  in uds_free_index()
1298 void uds_wait_for_idle_index(struct uds_index *index)  in uds_wait_for_idle_index()  argument
1300 	struct chapter_writer *writer = index->chapter_writer;  in uds_wait_for_idle_index()
1309 int uds_save_index(struct uds_index *index)  in uds_save_index()  argument
1313 	if (!index->need_to_save)  in uds_save_index()
1316 	uds_wait_for_idle_index(index);  in uds_save_index()
1317 	index->prev_save = index->last_save;  in uds_save_index()
1318 	index->last_save = ((index->newest_virtual_chapter == 0) ?  in uds_save_index()
1319 			    NO_LAST_SAVE : index->newest_virtual_chapter - 1);  in uds_save_index()
1320 	vdo_log_info("beginning save (vcn %llu)", (unsigned long long) index->last_save);  in uds_save_index()
1322 	result = uds_save_index_state(index->layout, index);  in uds_save_index()
1325 		index->last_save = index->prev_save;  in uds_save_index()
1327 		index->has_saved_open_chapter = true;  in uds_save_index()
1328 		index->need_to_save = false;  in uds_save_index()
1330 			     (unsigned long long) index->last_save);  in uds_save_index()
1336 int uds_replace_index_storage(struct uds_index *index, struct block_device *bdev)  in uds_replace_index_storage()  argument
1338 	return uds_replace_volume_storage(index->volume, index->layout, bdev);  in uds_replace_index_storage()
1342 void uds_get_index_stats(struct uds_index *index, struct uds_index_stats *counters)  in uds_get_index_stats()  argument
1346 	uds_get_volume_index_stats(index->volume_index, &stats);  in uds_get_index_stats()
1351 	counters->memory_used = (index->volume_index->memory_size +  in uds_get_index_stats()
1352 				 index->volume->cache_size +  in uds_get_index_stats()
1353 				 index->chapter_writer->memory_size);  in uds_get_index_stats()
1358 	struct uds_index *index = request->index;  in uds_enqueue_request()  local
1363 		if (index->triage_queue != NULL) {  in uds_enqueue_request()
1364 			queue = index->triage_queue;  in uds_enqueue_request()
1372 			uds_get_volume_index_zone(index->volume_index, &request->record_name);  in uds_enqueue_request()
1376 		queue = index->zone_queues[request->zone_number];  in uds_enqueue_request()