142 static void do_spring_cleaning(struct ce_array *ca)  in do_spring_cleaning()  argument
146 	for (i = 0; i < ca->n; i++) {  in do_spring_cleaning()
147 		u8 decay = DECAY(ca->array[i]);  in do_spring_cleaning()
154 		ca->array[i] &= ~(DECAY_MASK << COUNT_BITS);  in do_spring_cleaning()
155 		ca->array[i] |= (decay << COUNT_BITS);  in do_spring_cleaning()
157 	ca->decay_count = 0;  in do_spring_cleaning()
158 	ca->decays_done++;  in do_spring_cleaning()
186 static int __find_elem(struct ce_array *ca, u64 pfn, unsigned int *to)  in __find_elem()  argument
188 	int min = 0, max = ca->n - 1;  in __find_elem()
194 		this_pfn = PFN(ca->array[i]);  in __find_elem()
223 static int find_elem(struct ce_array *ca, u64 pfn, unsigned int *to)  in find_elem()  argument
227 	if (!ca->n) {  in find_elem()
231 	return __find_elem(ca, pfn, to);  in find_elem()
234 static void del_elem(struct ce_array *ca, int idx)  in del_elem()  argument
237 	if (ca->n - (idx + 1))  in del_elem()
238 		memmove((void *)&ca->array[idx],  in del_elem()
239 			(void *)&ca->array[idx + 1],  in del_elem()
240 			(ca->n - (idx + 1)) * sizeof(u64));  in del_elem()
242 	ca->n--;  in del_elem()
245 static u64 del_lru_elem_unlocked(struct ce_array *ca)  in del_lru_elem_unlocked()  argument
250 	for (i = 0; i < ca->n; i++) {  in del_lru_elem_unlocked()
251 		unsigned int this = FULL_COUNT(ca->array[i]);  in del_lru_elem_unlocked()
259 	del_elem(ca, min_idx);  in del_lru_elem_unlocked()
261 	return PFN(ca->array[min_idx]);  in del_lru_elem_unlocked()
270 	struct ce_array *ca = &ce_arr;  in del_lru_elem()  local
273 	if (!ca->n)  in del_lru_elem()
277 	pfn = del_lru_elem_unlocked(ca);  in del_lru_elem()
283 static bool sanity_check(struct ce_array *ca)  in sanity_check()  argument
289 	for (i = 0; i < ca->n; i++) {  in sanity_check()
290 		u64 this = PFN(ca->array[i]);  in sanity_check()
301 	pr_info("Sanity check dump:\n{ n: %d\n", ca->n);  in sanity_check()
302 	for (i = 0; i < ca->n; i++) {  in sanity_check()
303 		u64 this = PFN(ca->array[i]);  in sanity_check()
305 		pr_info(" %03d: [%016llx|%03llx]\n", i, this, FULL_COUNT(ca->array[i]));  in sanity_check()
323 	struct ce_array *ca = &ce_arr;  in cec_add_elem()  local
336 	ca->ces_entered++;  in cec_add_elem()
339 	if (ca->n == MAX_ELEMS)  in cec_add_elem()
340 		WARN_ON(!del_lru_elem_unlocked(ca));  in cec_add_elem()
342 	err = find_elem(ca, pfn, &to);  in cec_add_elem()
347 		memmove((void *)&ca->array[to + 1],  in cec_add_elem()
348 			(void *)&ca->array[to],  in cec_add_elem()
349 			(ca->n - to) * sizeof(u64));  in cec_add_elem()
351 		ca->array[to] = pfn << PAGE_SHIFT;  in cec_add_elem()
352 		ca->n++;  in cec_add_elem()
356 	ca->array[to] |= DECAY_MASK << COUNT_BITS;  in cec_add_elem()
357 	ca->array[to]++;  in cec_add_elem()
360 	count = COUNT(ca->array[to]);  in cec_add_elem()
362 		u64 pfn = ca->array[to] >> PAGE_SHIFT;  in cec_add_elem()
370 			ca->pfns_poisoned++;  in cec_add_elem()
373 		del_elem(ca, to);  in cec_add_elem()
384 	ca->decay_count++;  in cec_add_elem()
386 	if (ca->decay_count >= CLEAN_ELEMS)  in cec_add_elem()
387 		do_spring_cleaning(ca);  in cec_add_elem()
389 	WARN_ON_ONCE(sanity_check(ca));  in cec_add_elem()
449 	struct ce_array *ca = &ce_arr;  in array_show()  local
454 	seq_printf(m, "{ n: %d\n", ca->n);  in array_show()
455 	for (i = 0; i < ca->n; i++) {  in array_show()
456 		u64 this = PFN(ca->array[i]);  in array_show()
459 			   i, this, bins[DECAY(ca->array[i])], COUNT(ca->array[i]));  in array_show()
465 		   ca->ces_entered, ca->pfns_poisoned);  in array_show()
467 	seq_printf(m, "Flags: 0x%x\n", ca->flags);  in array_show()
470 	seq_printf(m, "Decays: %lld\n", ca->decays_done);  in array_show()