1 // SPDX-License-Identifier: GPL-2.0+
42 #define SCALE_FLAG "-ref-scale: "
85 // Number of readers, with -1 defaulting to about 75% of the CPUs.
86 torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
90 torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
152 	for (i = nloops; i >= 0; i--) {  in ref_rcu_read_section()
162 	for (i = nloops; i >= 0; i--) {  in ref_rcu_delay_section()
190 	for (i = nloops; i >= 0; i--) {  in srcu_ref_scale_read_section()
201 	for (i = nloops; i >= 0; i--) {  in srcu_ref_scale_delay_section()
223 	for (i = nloops; i >= 0; i--)  in rcu_tasks_ref_scale_read_section()
231 	for (i = nloops; i >= 0; i--)  in rcu_tasks_ref_scale_delay_section()
239 	.name		= "rcu-tasks"
257 	for (i = nloops; i >= 0; i--) {  in rcu_trace_ref_scale_read_section()
267 	for (i = nloops; i >= 0; i--) {  in rcu_trace_ref_scale_delay_section()
278 	.name		= "rcu-trace"
296 	for (i = nloops; i >= 0; i--) {  in ref_refcnt_section()
306 	for (i = nloops; i >= 0; i--) {  in ref_refcnt_delay_section()
333 	for (i = nloops; i >= 0; i--) {  in ref_rwlock_section()
343 	for (i = nloops; i >= 0; i--) {  in ref_rwlock_delay_section()
370 	for (i = nloops; i >= 0; i--) {  in ref_rwsem_section()
380 	for (i = nloops; i >= 0; i--) {  in ref_rwsem_delay_section()
402 	for (i = nloops; i >= 0; i--) {  in ref_lock_section()
414 	for (i = nloops; i >= 0; i--) {  in ref_lock_delay_section()
428 // Definitions for global irq-save spinlock
436 	for (i = nloops; i >= 0; i--) {  in ref_lock_irq_section()
449 	for (i = nloops; i >= 0; i--) {  in ref_lock_irq_delay_section()
460 	.name		= "lock-irq"
463 // Definitions acquire-release.
472 	for (i = nloops; i >= 0; i--) {  in ref_acqrel_section()
485 	for (i = nloops; i >= 0; i--) {  in ref_acqrel_delay_section()
507 	for (i = nloops; i >= 0; i--)  in ref_clock_section()
519 	for (i = nloops; i >= 0; i--) {  in ref_clock_delay_section()
530 	.name		= "clock"
539 	for (i = nloops; i >= 0; i--)  in ref_jiffies_section()
551 	for (i = nloops; i >= 0; i--) {  in ref_jiffies_delay_section()
586 // Conditionally acquire an explicit in-structure reference count.
589 	return atomic_inc_not_zero(&rtsp->rts_refctr);  in typesafe_ref_acquire()
592 // Unconditionally release an explicit in-structure reference count.
595 	if (!atomic_dec_return(&rtsp->rts_refctr)) {  in typesafe_ref_release()
596 		WRITE_ONCE(rtsp->a, rtsp->a + 1);  in typesafe_ref_release()
602 // Unconditionally acquire an explicit in-structure spinlock.
605 	spin_lock(&rtsp->rts_lock);  in typesafe_lock_acquire()
609 // Unconditionally release an explicit in-structure spinlock.
612 	spin_unlock(&rtsp->rts_lock);  in typesafe_lock_release()
616 // Unconditionally acquire an explicit in-structure sequence lock.
619 	*start = read_seqbegin(&rtsp->rts_seqlock);  in typesafe_seqlock_acquire()
623 // Conditionally release an explicit in-structure sequence lock.  Return
627 	return !read_seqretry(&rtsp->rts_seqlock, start);  in typesafe_seqlock_release()
630 // Do a read-side critical section with the specified delay in
642 	for (i = nloops; i >= 0; i--) {  in typesafe_delay_section()
649 		a = READ_ONCE(rtsp->a);  in typesafe_delay_section()
654 		if (a != READ_ONCE(rtsp->a)) {  in typesafe_delay_section()
660 		b = READ_ONCE(rtsp->a);  in typesafe_delay_section()
661 		// Remember, seqlock read-side release can fail.  in typesafe_delay_section()
666 		WARN_ONCE(a != b, "Re-read of ->a changed from %u to %u.\n", a, b);  in typesafe_delay_section()
667 		b = rtsp->b;  in typesafe_delay_section()
690 	atomic_set(&rtsp->rts_refctr, 1);  in typesafe_alloc_one()
691 	WRITE_ONCE(rtsp->a, rtsp->a + 1);  in typesafe_alloc_one()
692 	WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a);  in typesafe_alloc_one()
696 // Slab-allocator constructor for refscale_typesafe structures created
702 	spin_lock_init(&rtsp->rts_lock);  in refscale_typesafe_ctor()
703 	seqlock_init(&rtsp->rts_seqlock);  in refscale_typesafe_ctor()
705 	rtsp->a = torture_random(this_cpu_ptr(&refscale_rand));  in refscale_typesafe_ctor()
725 		si = -si * nr_cpu_ids;  in typesafe_init()
799 		cur_ops->readsection(loops);  in rcu_scale_one_reader()
801 		cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);  in rcu_scale_one_reader()
804 // Reader kthread.  Repeatedly does empty RCU read-side
805 // critical section, minimizing update-side interference.
813 	s64 duration;  in ref_scale_reader()  local
825 	wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||  in ref_scale_reader()
834 	WRITE_ONCE(rt->start_reader, 0);  in ref_scale_reader()
842 	// To reduce noise, do an initial cache-warming invocation, check  in ref_scale_reader()
855 	duration = ktime_get_mono_fast_ns() - start;  in ref_scale_reader()
858 	rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;  in ref_scale_reader()
859 	// To reduce runtime-skew noise, do maintain-load invocations until  in ref_scale_reader()
886 		rt->last_duration_ns = 0;  in reset_readers()
904 	seq_buf_printf(&s, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",  in process_durations()
918 		seq_buf_printf(&s, "%d: %llu\t", i, rt->last_duration_ns);  in process_durations()
920 		sum += rt->last_duration_ns;  in process_durations()
992 	SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");  in main_func()
994 	pr_alert("Runs\tTime(ns)\n");  in main_func()
1032 …"--- %s:  verbose=%d verbose_batched=%d shutdown=%d holdoff=%d lookup_instances=%ld loops=%ld nrea…  in ref_scale_print_module_parms()
1059 	// Do scale-type-specific cleanup operations.  in ref_scale_cleanup()
1060 	if (cur_ops->cleanup != NULL)  in ref_scale_cleanup()
1061 		cur_ops->cleanup();  in ref_scale_cleanup()
1076 	return -EINVAL;  in ref_scale_shutdown()
1091 		return -EBUSY;  in ref_scale_init()
1095 		if (strcmp(scale_type, cur_ops->name) == 0)  in ref_scale_init()
1099 		pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);  in ref_scale_init()
1100 		pr_alert("rcu-scale types:");  in ref_scale_init()
1102 			pr_cont(" %s", scale_ops[i]->name);  in ref_scale_init()
1104 		firsterr = -EINVAL;  in ref_scale_init()
1108 	if (cur_ops->init)  in ref_scale_init()
1109 		if (!cur_ops->init()) {  in ref_scale_init()
1110 			firsterr = -EUCLEAN;  in ref_scale_init()
1139 		firsterr = -ENOMEM;  in ref_scale_init()