Lines Matching refs:limiter
123 struct limiter;
124 typedef void (*assigner_fn)(struct limiter *limiter);
127 struct limiter { struct
166 struct limiter limiter; member
168 struct limiter discard_limiter;
232 if (pool->limiter.busy > 0) in check_for_drain_complete_locked()
238 return (bio_list_empty(&pool->limiter.new_waiters) && in check_for_drain_complete_locked()
585 static void assign_data_vio(struct limiter *limiter, struct data_vio *data_vio) in assign_data_vio() argument
587 struct bio *bio = bio_list_pop(limiter->permitted_waiters); in assign_data_vio()
589 launch_bio(limiter->pool->completion.vdo, data_vio, bio); in assign_data_vio()
590 limiter->wake_count++; in assign_data_vio()
592 bio = bio_list_peek(limiter->permitted_waiters); in assign_data_vio()
593 limiter->arrival = ((bio == NULL) ? U64_MAX : get_arrival_time(bio)); in assign_data_vio()
596 static void assign_discard_permit(struct limiter *limiter) in assign_discard_permit() argument
598 struct bio *bio = bio_list_pop(&limiter->waiters); in assign_discard_permit()
600 if (limiter->arrival == U64_MAX) in assign_discard_permit()
601 limiter->arrival = get_arrival_time(bio); in assign_discard_permit()
603 bio_list_add(limiter->permitted_waiters, bio); in assign_discard_permit()
606 static void get_waiters(struct limiter *limiter) in get_waiters() argument
608 bio_list_merge_init(&limiter->waiters, &limiter->new_waiters); in get_waiters()
620 static void assign_data_vio_to_waiter(struct limiter *limiter) in assign_data_vio_to_waiter() argument
622 assign_data_vio(limiter, get_available_data_vio(limiter->pool)); in assign_data_vio_to_waiter()
625 static void update_limiter(struct limiter *limiter) in update_limiter() argument
627 struct bio_list *waiters = &limiter->waiters; in update_limiter()
628 data_vio_count_t available = limiter->limit - limiter->busy; in update_limiter()
630 VDO_ASSERT_LOG_ONLY((limiter->release_count <= limiter->busy), in update_limiter()
632 limiter->release_count, limiter->busy); in update_limiter()
634 get_waiters(limiter); in update_limiter()
635 for (; (limiter->release_count > 0) && !bio_list_empty(waiters); limiter->release_count--) in update_limiter()
636 limiter->assigner(limiter); in update_limiter()
638 if (limiter->release_count > 0) { in update_limiter()
639 WRITE_ONCE(limiter->busy, limiter->busy - limiter->release_count); in update_limiter()
640 limiter->release_count = 0; in update_limiter()
645 limiter->assigner(limiter); in update_limiter()
647 WRITE_ONCE(limiter->busy, limiter->limit - available); in update_limiter()
648 if (limiter->max_busy < limiter->busy) in update_limiter()
649 WRITE_ONCE(limiter->max_busy, limiter->busy); in update_limiter()
683 if (pool->limiter.arrival < pool->discard_limiter.arrival) { in reuse_or_release_resources()
684 assign_data_vio(&pool->limiter, data_vio); in reuse_or_release_resources()
689 pool->limiter.release_count++; in reuse_or_release_resources()
709 get_waiters(&pool->limiter); in process_release_callback()
712 if (pool->limiter.arrival == U64_MAX) { in process_release_callback()
713 struct bio *bio = bio_list_peek(&pool->limiter.waiters); in process_release_callback()
716 pool->limiter.arrival = get_arrival_time(bio); in process_release_callback()
741 update_limiter(&pool->limiter); in process_release_callback()
742 to_wake = pool->limiter.wake_count; in process_release_callback()
743 pool->limiter.wake_count = 0; in process_release_callback()
758 wake_up_nr(&pool->limiter.blocked_threads, to_wake); in process_release_callback()
769 static void initialize_limiter(struct limiter *limiter, struct data_vio_pool *pool, in initialize_limiter() argument
772 limiter->pool = pool; in initialize_limiter()
773 limiter->assigner = assigner; in initialize_limiter()
774 limiter->limit = limit; in initialize_limiter()
775 limiter->arrival = U64_MAX; in initialize_limiter()
776 init_waitqueue_head(&limiter->blocked_threads); in initialize_limiter()
858 initialize_limiter(&pool->limiter, pool, assign_data_vio_to_waiter, pool_size); in make_data_vio_pool()
859 pool->limiter.permitted_waiters = &pool->limiter.waiters; in make_data_vio_pool()
911 VDO_ASSERT_LOG_ONLY((pool->limiter.busy == 0), in free_data_vio_pool()
913 pool->limiter.busy); in free_data_vio_pool()
914 VDO_ASSERT_LOG_ONLY((bio_list_empty(&pool->limiter.waiters) && in free_data_vio_pool()
915 bio_list_empty(&pool->limiter.new_waiters)), in free_data_vio_pool()
931 static bool acquire_permit(struct limiter *limiter) in acquire_permit() argument
933 if (limiter->busy >= limiter->limit) in acquire_permit()
936 WRITE_ONCE(limiter->busy, limiter->busy + 1); in acquire_permit()
937 if (limiter->max_busy < limiter->busy) in acquire_permit()
938 WRITE_ONCE(limiter->max_busy, limiter->busy); in acquire_permit()
942 static void wait_permit(struct limiter *limiter, struct bio *bio) in wait_permit() argument
943 __releases(&limiter->pool->lock) in wait_permit()
947 bio_list_add(&limiter->new_waiters, bio); in wait_permit()
948 prepare_to_wait_exclusive(&limiter->blocked_threads, &wait, in wait_permit()
950 spin_unlock(&limiter->pool->lock); in wait_permit()
952 finish_wait(&limiter->blocked_threads, &wait); in wait_permit()
975 if (!acquire_permit(&pool->limiter)) { in vdo_launch_bio()
976 wait_permit(&pool->limiter, bio); in vdo_launch_bio()
1026 static void dump_limiter(const char *name, struct limiter *limiter) in dump_limiter() argument
1028 vdo_log_info("%s: %u of %u busy (max %u), %s", name, limiter->busy, in dump_limiter()
1029 limiter->limit, limiter->max_busy, in dump_limiter()
1030 ((bio_list_empty(&limiter->waiters) && in dump_limiter()
1031 bio_list_empty(&limiter->new_waiters)) ? in dump_limiter()
1052 dump_limiter("data_vios", &pool->limiter); in dump_data_vio_pool()
1058 for (i = 0; i < pool->limiter.limit; i++) { in dump_data_vio_pool()
1108 return READ_ONCE(pool->limiter.busy); in get_data_vio_pool_active_requests()
1113 return READ_ONCE(pool->limiter.limit); in get_data_vio_pool_request_limit()
1118 return READ_ONCE(pool->limiter.max_busy); in get_data_vio_pool_maximum_requests()