1 // SPDX-License-Identifier: GPL-2.0-only
6 #include "index-session.h"
11 #include "memory-alloc.h"
12 #include "time-utils.h"
14 #include "funnel-requestqueue.h"
16 #include "index-layout.h"
21  * index through a set of descriptive flags. The request_mutex must be notified whenever a
22  * non-transient state flag is cleared. The request_mutex is also used to count the number of
65 	mutex_lock(&index_session->request_mutex);  in release_index_session()
66 	if (--index_session->request_count == 0)  in release_index_session()
67 		uds_broadcast_cond(&index_session->request_cond);  in release_index_session()
68 	mutex_unlock(&index_session->request_mutex);  in release_index_session()
80 	mutex_lock(&index_session->request_mutex);  in get_index_session()
81 	index_session->request_count++;  in get_index_session()
82 	state = index_session->state;  in get_index_session()
83 	mutex_unlock(&index_session->request_mutex);  in get_index_session()
92 		result = -EBUSY;  in get_index_session()
106 	if (request->callback == NULL) {  in uds_launch_request()
108 		return -EINVAL;  in uds_launch_request()
111 	switch (request->type) {  in uds_launch_request()
120 		return -EINVAL;  in uds_launch_request()
125 		sizeof(struct uds_request) - offsetof(struct uds_request, zone_number);  in uds_launch_request()
127 	memset((char *) request + sizeof(*request) - internal_size, 0, internal_size);  in uds_launch_request()
129 	result = get_index_session(request->session);  in uds_launch_request()
133 	request->found = false;  in uds_launch_request()
134 	request->unbatched = false;  in uds_launch_request()
135 	request->index = request->session->index;  in uds_launch_request()
143 	if (request->status != UDS_SUCCESS) {  in enter_callback_stage()
145 		mutex_lock(&request->session->request_mutex);  in enter_callback_stage()
146 		request->session->state |= IS_FLAG_DISABLED;  in enter_callback_stage()
147 		mutex_unlock(&request->session->request_mutex);  in enter_callback_stage()
150 	uds_request_queue_enqueue(request->session->callback_queue, request);  in enter_callback_stage()
160 	struct session_stats *session_stats = &request->session->stats;  in update_session_stats()
162 	count_once(&session_stats->requests);  in update_session_stats()
164 	switch (request->type) {  in update_session_stats()
166 		if (request->found)  in update_session_stats()
167 			count_once(&session_stats->posts_found);  in update_session_stats()
169 			count_once(&session_stats->posts_not_found);  in update_session_stats()
171 		if (request->location == UDS_LOCATION_IN_OPEN_CHAPTER)  in update_session_stats()
172 			count_once(&session_stats->posts_found_open_chapter);  in update_session_stats()
173 		else if (request->location == UDS_LOCATION_IN_DENSE)  in update_session_stats()
174 			count_once(&session_stats->posts_found_dense);  in update_session_stats()
175 		else if (request->location == UDS_LOCATION_IN_SPARSE)  in update_session_stats()
176 			count_once(&session_stats->posts_found_sparse);  in update_session_stats()
180 		if (request->found)  in update_session_stats()
181 			count_once(&session_stats->updates_found);  in update_session_stats()
183 			count_once(&session_stats->updates_not_found);  in update_session_stats()
187 		if (request->found)  in update_session_stats()
188 			count_once(&session_stats->deletions_found);  in update_session_stats()
190 			count_once(&session_stats->deletions_not_found);  in update_session_stats()
195 		if (request->found)  in update_session_stats()
196 			count_once(&session_stats->queries_found);  in update_session_stats()
198 			count_once(&session_stats->queries_not_found);  in update_session_stats()
202 		request->status = VDO_ASSERT(false, "unknown request type: %d",  in update_session_stats()
203 					     request->type);  in update_session_stats()
209 	struct uds_index_session *index_session = request->session;  in handle_callbacks()
211 	if (request->status == UDS_SUCCESS)  in handle_callbacks()
214 	request->status = uds_status_to_errno(request->status);  in handle_callbacks()
215 	request->callback(request);  in handle_callbacks()
228 	mutex_init(&session->request_mutex);  in make_empty_index_session()
229 	uds_init_cond(&session->request_cond);  in make_empty_index_session()
230 	mutex_init(&session->load_context.mutex);  in make_empty_index_session()
231 	uds_init_cond(&session->load_context.cond);  in make_empty_index_session()
234 					&session->callback_queue);  in make_empty_index_session()
248 		return -EINVAL;  in uds_create_index_session()
258 	mutex_lock(&index_session->request_mutex);  in start_loading_index_session()
259 	if (index_session->state & IS_FLAG_SUSPENDED) {  in start_loading_index_session()
261 		result = -EBUSY;  in start_loading_index_session()
262 	} else if (index_session->state != 0) {  in start_loading_index_session()
264 		result = -EBUSY;  in start_loading_index_session()
266 		index_session->state |= IS_FLAG_LOADING;  in start_loading_index_session()
269 	mutex_unlock(&index_session->request_mutex);  in start_loading_index_session()
276 	mutex_lock(&index_session->request_mutex);  in finish_loading_index_session()
277 	index_session->state &= ~IS_FLAG_LOADING;  in finish_loading_index_session()
279 		index_session->state |= IS_FLAG_LOADED;  in finish_loading_index_session()
281 	uds_broadcast_cond(&index_session->request_cond);  in finish_loading_index_session()
282 	mutex_unlock(&index_session->request_mutex);  in finish_loading_index_session()
291 	result = uds_make_configuration(&index_session->parameters, &config);  in initialize_index_session()
297 	memset(&index_session->stats, 0, sizeof(index_session->stats));  in initialize_index_session()
298 	result = uds_make_index(config, open_type, &index_session->load_context,  in initialize_index_session()
299 				enter_callback_stage, &index_session->index);  in initialize_index_session()
336 		return -EINVAL;  in uds_open_index()
338 	if (parameters->bdev == NULL) {  in uds_open_index()
340 		return -EINVAL;  in uds_open_index()
344 		return -EINVAL;  in uds_open_index()
351 	session->parameters = *parameters;  in uds_open_index()
352 	format_dev_t(name, parameters->bdev->bd_dev);  in uds_open_index()
366 	mutex_lock(&index_session->request_mutex);  in wait_for_no_requests_in_progress()
367 	while (index_session->request_count > 0) {  in wait_for_no_requests_in_progress()
368 		uds_wait_cond(&index_session->request_cond,  in wait_for_no_requests_in_progress()
369 			      &index_session->request_mutex);  in wait_for_no_requests_in_progress()
371 	mutex_unlock(&index_session->request_mutex);  in wait_for_no_requests_in_progress()
377 	return uds_save_index(index_session->index);  in save_index()
382 	mutex_lock(&session->load_context.mutex);  in suspend_rebuild()
383 	switch (session->load_context.status) {  in suspend_rebuild()
385 		session->load_context.status = INDEX_SUSPENDING;  in suspend_rebuild()
388 		while ((session->load_context.status != INDEX_SUSPENDED) &&  in suspend_rebuild()
389 		       (session->load_context.status != INDEX_READY)) {  in suspend_rebuild()
390 			uds_wait_cond(&session->load_context.cond,  in suspend_rebuild()
391 				      &session->load_context.mutex);  in suspend_rebuild()
406 				    session->load_context.status);  in suspend_rebuild()
409 	mutex_unlock(&session->load_context.mutex);  in suspend_rebuild()
423 	mutex_lock(&session->request_mutex);  in uds_suspend_index_session()
424 	while (session->state & IS_FLAG_CLOSING)  in uds_suspend_index_session()
425 		uds_wait_cond(&session->request_cond, &session->request_mutex);  in uds_suspend_index_session()
427 	if ((session->state & IS_FLAG_WAITING) || (session->state & IS_FLAG_DESTROYING)) {  in uds_suspend_index_session()
430 		result = -EBUSY;  in uds_suspend_index_session()
431 	} else if (session->state & IS_FLAG_SUSPENDED) {  in uds_suspend_index_session()
433 	} else if (session->state & IS_FLAG_LOADING) {  in uds_suspend_index_session()
434 		session->state |= IS_FLAG_WAITING;  in uds_suspend_index_session()
436 	} else if (session->state & IS_FLAG_LOADED) {  in uds_suspend_index_session()
437 		session->state |= IS_FLAG_WAITING;  in uds_suspend_index_session()
440 		session->state |= IS_FLAG_SUSPENDED;  in uds_suspend_index_session()
441 		uds_broadcast_cond(&session->request_cond);  in uds_suspend_index_session()
443 	mutex_unlock(&session->request_mutex);  in uds_suspend_index_session()
455 	mutex_lock(&session->request_mutex);  in uds_suspend_index_session()
456 	session->state &= ~IS_FLAG_WAITING;  in uds_suspend_index_session()
457 	session->state |= IS_FLAG_SUSPENDED;  in uds_suspend_index_session()
458 	uds_broadcast_cond(&session->request_cond);  in uds_suspend_index_session()
459 	mutex_unlock(&session->request_mutex);  in uds_suspend_index_session()
467 	result = uds_replace_index_storage(session->index, bdev);  in replace_device()
471 	session->parameters.bdev = bdev;  in replace_device()
486 	mutex_lock(&session->request_mutex);  in uds_resume_index_session()
487 	if (session->state & IS_FLAG_WAITING) {  in uds_resume_index_session()
490 		result = -EBUSY;  in uds_resume_index_session()
491 	} else if (!(session->state & IS_FLAG_SUSPENDED)) {  in uds_resume_index_session()
496 		session->state |= IS_FLAG_WAITING;  in uds_resume_index_session()
497 		if (session->state & IS_FLAG_LOADING)  in uds_resume_index_session()
500 	mutex_unlock(&session->request_mutex);  in uds_resume_index_session()
505 	if ((session->index != NULL) && (bdev != session->parameters.bdev)) {  in uds_resume_index_session()
508 			mutex_lock(&session->request_mutex);  in uds_resume_index_session()
509 			session->state &= ~IS_FLAG_WAITING;  in uds_resume_index_session()
510 			uds_broadcast_cond(&session->request_cond);  in uds_resume_index_session()
511 			mutex_unlock(&session->request_mutex);  in uds_resume_index_session()
517 		mutex_lock(&session->load_context.mutex);  in uds_resume_index_session()
518 		switch (session->load_context.status) {  in uds_resume_index_session()
520 			session->load_context.status = INDEX_OPENING;  in uds_resume_index_session()
522 			uds_broadcast_cond(&session->load_context.cond);  in uds_resume_index_session()
535 					    session->load_context.status);  in uds_resume_index_session()
538 		mutex_unlock(&session->load_context.mutex);  in uds_resume_index_session()
541 	mutex_lock(&session->request_mutex);  in uds_resume_index_session()
542 	session->state &= ~IS_FLAG_WAITING;  in uds_resume_index_session()
543 	session->state &= ~IS_FLAG_SUSPENDED;  in uds_resume_index_session()
544 	uds_broadcast_cond(&session->request_cond);  in uds_resume_index_session()
545 	mutex_unlock(&session->request_mutex);  in uds_resume_index_session()
553 	struct uds_index *index = index_session->index;  in save_and_free_index()
558 	mutex_lock(&index_session->request_mutex);  in save_and_free_index()
559 	suspended = (index_session->state & IS_FLAG_SUSPENDED);  in save_and_free_index()
560 	mutex_unlock(&index_session->request_mutex);  in save_and_free_index()
569 	index_session->index = NULL;  in save_and_free_index()
575 	mutex_lock(&index_session->load_context.mutex);  in save_and_free_index()
576 	index_session->load_context.status = INDEX_OPENING;  in save_and_free_index()
577 	mutex_unlock(&index_session->load_context.mutex);  in save_and_free_index()
579 	mutex_lock(&index_session->request_mutex);  in save_and_free_index()
581 	index_session->state &= IS_FLAG_SUSPENDED;  in save_and_free_index()
582 	mutex_unlock(&index_session->request_mutex);  in save_and_free_index()
593 	mutex_lock(&index_session->request_mutex);  in uds_close_index()
594 	while ((index_session->state & IS_FLAG_WAITING) ||  in uds_close_index()
595 	       (index_session->state & IS_FLAG_CLOSING)) {  in uds_close_index()
596 		uds_wait_cond(&index_session->request_cond,  in uds_close_index()
597 			      &index_session->request_mutex);  in uds_close_index()
600 	if (index_session->state & IS_FLAG_SUSPENDED) {  in uds_close_index()
602 		result = -EBUSY;  in uds_close_index()
603 	} else if ((index_session->state & IS_FLAG_DESTROYING) ||  in uds_close_index()
604 		   !(index_session->state & IS_FLAG_LOADED)) {  in uds_close_index()
608 		index_session->state |= IS_FLAG_CLOSING;  in uds_close_index()
610 	mutex_unlock(&index_session->request_mutex);  in uds_close_index()
619 	mutex_lock(&index_session->request_mutex);  in uds_close_index()
620 	index_session->state &= ~IS_FLAG_CLOSING;  in uds_close_index()
621 	uds_broadcast_cond(&index_session->request_cond);  in uds_close_index()
622 	mutex_unlock(&index_session->request_mutex);  in uds_close_index()
635 	mutex_lock(&index_session->request_mutex);  in uds_destroy_index_session()
636 	while ((index_session->state & IS_FLAG_WAITING) ||  in uds_destroy_index_session()
637 	       (index_session->state & IS_FLAG_CLOSING)) {  in uds_destroy_index_session()
638 		uds_wait_cond(&index_session->request_cond,  in uds_destroy_index_session()
639 			      &index_session->request_mutex);  in uds_destroy_index_session()
642 	if (index_session->state & IS_FLAG_DESTROYING) {  in uds_destroy_index_session()
643 		mutex_unlock(&index_session->request_mutex);  in uds_destroy_index_session()
645 		return -EBUSY;  in uds_destroy_index_session()
648 	index_session->state |= IS_FLAG_DESTROYING;  in uds_destroy_index_session()
649 	load_pending = ((index_session->state & IS_FLAG_LOADING) &&  in uds_destroy_index_session()
650 			(index_session->state & IS_FLAG_SUSPENDED));  in uds_destroy_index_session()
651 	mutex_unlock(&index_session->request_mutex);  in uds_destroy_index_session()
655 		mutex_lock(&index_session->load_context.mutex);  in uds_destroy_index_session()
656 		if (index_session->load_context.status == INDEX_SUSPENDED) {  in uds_destroy_index_session()
657 			index_session->load_context.status = INDEX_FREEING;  in uds_destroy_index_session()
658 			uds_broadcast_cond(&index_session->load_context.cond);  in uds_destroy_index_session()
660 		mutex_unlock(&index_session->load_context.mutex);  in uds_destroy_index_session()
663 		mutex_lock(&index_session->request_mutex);  in uds_destroy_index_session()
664 		while (index_session->state & IS_FLAG_LOADING) {  in uds_destroy_index_session()
665 			uds_wait_cond(&index_session->request_cond,  in uds_destroy_index_session()
666 				      &index_session->request_mutex);  in uds_destroy_index_session()
668 		mutex_unlock(&index_session->request_mutex);  in uds_destroy_index_session()
673 	uds_request_queue_finish(index_session->callback_queue);  in uds_destroy_index_session()
674 	index_session->callback_queue = NULL;  in uds_destroy_index_session()
684 	uds_wait_for_idle_index(index_session->index);  in uds_flush_index_session()
688 /* Statistics collection is intended to be thread-safe. */
692 	const struct session_stats *session_stats = &index_session->stats;  in collect_stats()
694 	stats->current_time = ktime_to_seconds(current_time_ns(CLOCK_REALTIME));  in collect_stats()
695 	stats->posts_found = READ_ONCE(session_stats->posts_found);  in collect_stats()
696 	stats->in_memory_posts_found = READ_ONCE(session_stats->posts_found_open_chapter);  in collect_stats()
697 	stats->dense_posts_found = READ_ONCE(session_stats->posts_found_dense);  in collect_stats()
698 	stats->sparse_posts_found = READ_ONCE(session_stats->posts_found_sparse);  in collect_stats()
699 	stats->posts_not_found = READ_ONCE(session_stats->posts_not_found);  in collect_stats()
700 	stats->updates_found = READ_ONCE(session_stats->updates_found);  in collect_stats()
701 	stats->updates_not_found = READ_ONCE(session_stats->updates_not_found);  in collect_stats()
702 	stats->deletions_found = READ_ONCE(session_stats->deletions_found);  in collect_stats()
703 	stats->deletions_not_found = READ_ONCE(session_stats->deletions_not_found);  in collect_stats()
704 	stats->queries_found = READ_ONCE(session_stats->queries_found);  in collect_stats()
705 	stats->queries_not_found = READ_ONCE(session_stats->queries_not_found);  in collect_stats()
706 	stats->requests = READ_ONCE(session_stats->requests);  in collect_stats()
714 		return -EINVAL;  in uds_get_index_session_stats()
718 	if (index_session->index != NULL) {  in uds_get_index_session_stats()
719 		uds_get_index_stats(index_session->index, stats);  in uds_get_index_session_stats()
721 		stats->entries_indexed = 0;  in uds_get_index_session_stats()
722 		stats->memory_used = 0;  in uds_get_index_session_stats()
723 		stats->collisions = 0;  in uds_get_index_session_stats()
724 		stats->entries_discarded = 0;  in uds_get_index_session_stats()
734 	prepare_to_wait(&cv->wait_queue, &__wait, TASK_IDLE);  in uds_wait_cond()
737 	finish_wait(&cv->wait_queue, &__wait);  in uds_wait_cond()