86 static inline u32 map_to_page_number(struct index_geometry *geometry, u32 physical_page)  in map_to_page_number()  argument
88 	return (physical_page - HEADER_PAGES_PER_VOLUME) % geometry->pages_per_chapter;  in map_to_page_number()
91 static inline u32 map_to_chapter_number(struct index_geometry *geometry, u32 physical_page)  in map_to_chapter_number()  argument
93 	return (physical_page - HEADER_PAGES_PER_VOLUME) / geometry->pages_per_chapter;  in map_to_chapter_number()
96 static inline bool is_record_page(struct index_geometry *geometry, u32 physical_page)  in is_record_page()  argument
98 	return map_to_page_number(geometry, physical_page) >= geometry->index_pages_per_chapter;  in is_record_page()
129 static void begin_pending_search(struct page_cache *cache, u32 physical_page,  in begin_pending_search()  argument
135 	invalidate_counter.page = physical_page;  in begin_pending_search()
167 static void wait_for_pending_searches(struct page_cache *cache, u32 physical_page)  in wait_for_pending_searches()  argument
184 		    (initial_counters[i].page == physical_page)) {  in wait_for_pending_searches()
208 	page->physical_page = cache->indexable_pages;  in clear_cache_page()
245 	if (page->physical_page != cache->indexable_pages) {  in select_victim_in_cache()
246 		WRITE_ONCE(cache->index[page->physical_page], cache->cache_slots);  in select_victim_in_cache()
247 		wait_for_pending_searches(cache, page->physical_page);  in select_victim_in_cache()
256 static int put_page_in_cache(struct page_cache *cache, u32 physical_page,  in put_page_in_cache()  argument
266 	page->physical_page = physical_page;  in put_page_in_cache()
278 	WRITE_ONCE(cache->index[physical_page], page - cache->cache);  in put_page_in_cache()
282 static void cancel_page_in_cache(struct page_cache *cache, u32 physical_page,  in cancel_page_in_cache()  argument
296 	WRITE_ONCE(cache->index[physical_page], cache->cache_slots);  in cancel_page_in_cache()
315 			 u32 physical_page)  in enqueue_read()  argument
322 	if ((cache->index[physical_page] & VOLUME_CACHE_QUEUED_FLAG) == 0) {  in enqueue_read()
328 		cache->read_queue[last].physical_page = physical_page;  in enqueue_read()
335 		WRITE_ONCE(cache->index[physical_page],  in enqueue_read()
341 		read_queue_index = cache->index[physical_page] & ~VOLUME_CACHE_QUEUED_FLAG;  in enqueue_read()
356 			      u32 physical_page)  in enqueue_page_read()  argument
359 	while (!enqueue_read(&volume->page_cache, request, physical_page)) {  in enqueue_page_read()
384 	index_value = cache->index[entry->physical_page];  in reserve_read_queue_entry()
388 		WRITE_ONCE(cache->index[entry->physical_page], cache->cache_slots);  in reserve_read_queue_entry()
459 static int initialize_index_page(const struct volume *volume, u32 physical_page,  in initialize_index_page()  argument
462 	u32 chapter = map_to_chapter_number(volume->geometry, physical_page);  in initialize_index_page()
463 	u32 index_page_number = map_to_page_number(volume->geometry, physical_page);  in initialize_index_page()
507 		       struct uds_request *request, u32 physical_page)  in search_page()  argument
513 	if (is_record_page(volume->geometry, physical_page)) {  in search_page()
543 	u32 page_number = entry->physical_page;  in process_entry()
644 static void get_page_and_index(struct page_cache *cache, u32 physical_page,  in get_page_and_index()  argument
662 	index_value = READ_ONCE(cache->index[physical_page]);  in get_page_and_index()
681 static void get_page_from_cache(struct page_cache *cache, u32 physical_page,  in get_page_from_cache()  argument
690 	get_page_and_index(cache, physical_page, &queue_index, page);  in get_page_from_cache()
693 static int read_page_locked(struct volume *volume, u32 physical_page,  in read_page_locked()  argument
701 	page_data = dm_bufio_read(volume->client, physical_page, &page->buffer);  in read_page_locked()
706 					 physical_page);  in read_page_locked()
707 		cancel_page_in_cache(&volume->page_cache, physical_page, page);  in read_page_locked()
711 	if (!is_record_page(volume->geometry, physical_page)) {  in read_page_locked()
712 		result = initialize_index_page(volume, physical_page, page);  in read_page_locked()
715 				vdo_log_warning("Corrupt index page %u", physical_page);  in read_page_locked()
716 			cancel_page_in_cache(&volume->page_cache, physical_page, page);  in read_page_locked()
721 	result = put_page_in_cache(&volume->page_cache, physical_page, page);  in read_page_locked()
723 		vdo_log_warning("Error putting page %u in cache", physical_page);  in read_page_locked()
724 		cancel_page_in_cache(&volume->page_cache, physical_page, page);  in read_page_locked()
733 static int get_volume_page_locked(struct volume *volume, u32 physical_page,  in get_volume_page_locked()  argument
739 	get_page_from_cache(&volume->page_cache, physical_page, &page);  in get_volume_page_locked()
741 		result = read_page_locked(volume, physical_page, &page);  in get_volume_page_locked()
754 				     u32 physical_page, struct cached_page **page_ptr)  in get_volume_page_protected()  argument
758 	get_page_from_cache(&volume->page_cache, physical_page, &page);  in get_volume_page_protected()
781 	get_page_from_cache(&volume->page_cache, physical_page, &page);  in get_volume_page_protected()
783 		enqueue_page_read(volume, request, physical_page);  in get_volume_page_protected()
790 		begin_pending_search(&volume->page_cache, physical_page,  in get_volume_page_protected()
800 	begin_pending_search(&volume->page_cache, physical_page, request->zone_number);  in get_volume_page_protected()
810 	u32 physical_page = map_to_physical_page(volume->geometry, chapter, page_number);  in get_volume_page()  local
813 	result = get_volume_page_locked(volume, physical_page, page_ptr);  in get_volume_page()
852 	u32 physical_page = map_to_physical_page(volume->geometry, chapter,  in search_cached_index_page()  local
861 	begin_pending_search(&volume->page_cache, physical_page, request->zone_number);  in search_cached_index_page()
863 	result = get_volume_page_protected(volume, request, physical_page, &page);  in search_cached_index_page()
886 	u32 physical_page, page_number;  in uds_search_cached_record_page()  local
900 	physical_page = map_to_physical_page(volume->geometry, chapter, page_number);  in uds_search_cached_record_page()
908 	begin_pending_search(&volume->page_cache, physical_page, request->zone_number);  in uds_search_cached_record_page()
910 	result = get_volume_page_protected(volume, request, physical_page, &record_page);  in uds_search_cached_record_page()
927 	u32 physical_page = map_to_physical_page(geometry, chapter, 0);  in uds_prefetch_volume_chapter()  local
929 	dm_bufio_prefetch(volume->client, physical_page, geometry->pages_per_chapter);  in uds_prefetch_volume_chapter()
940 	u32 physical_page = map_to_physical_page(geometry, physical_chapter, 0);  in uds_read_chapter_index_from_volume()  local
942 	dm_bufio_prefetch(volume->client, physical_page, geometry->index_pages_per_chapter);  in uds_read_chapter_index_from_volume()
946 		index_page = dm_bufio_read(volume->client, physical_page + i,  in uds_read_chapter_index_from_volume()
952 						 physical_page);  in uds_read_chapter_index_from_volume()
1030 static void invalidate_page(struct page_cache *cache, u32 physical_page)  in invalidate_page()  argument
1036 	get_page_and_index(cache, physical_page, &queue_index, &page);  in invalidate_page()
1038 		WRITE_ONCE(cache->index[page->physical_page], cache->cache_slots);  in invalidate_page()
1039 		wait_for_pending_searches(cache, page->physical_page);  in invalidate_page()
1070 	u32 physical_page =  in donate_index_page_locked()  local
1081 		cancel_page_in_cache(&volume->page_cache, physical_page, page);  in donate_index_page_locked()
1085 	result = put_page_in_cache(&volume->page_cache, physical_page, page);  in donate_index_page_locked()
1087 		vdo_log_warning("Error putting page %u in cache", physical_page);  in donate_index_page_locked()
1088 		cancel_page_in_cache(&volume->page_cache, physical_page, page);  in donate_index_page_locked()
1108 		u32 physical_page = first_index_page + index_page_number;  in write_index_pages()  local
1113 		page_data = dm_bufio_new(volume->client, physical_page, &page_buffer);  in write_index_pages()
1219 		u32 physical_page = first_record_page + record_page_number;  in write_record_pages()  local
1222 		page_data = dm_bufio_new(volume->client, physical_page, &page_buffer);  in write_record_pages()