Lines Matching +full:diff +full:- +full:channels
1 // SPDX-License-Identifier: GPL-2.0-only
12 #include <linux/dma-mapping.h>
44 "Maximum number of channels to use (default: all)");
54 "dmatest 0-memcpy 1-memset (default: 0)");
69 "Pass -1 for infinite timeout");
83 static int alignment = -1;
85 …M_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))");
96 * struct dmatest_params - test parameters.
102 * @max_channels: maximum number of channels to use
106 * @timeout: transfer timeout in msec, -1 for infinite timeout
132 * struct dmatest_info - test information.
134 * @channels: channels under test
135 * @nr_channels: number of channels under test
145 struct list_head channels; member
151 .channels = LIST_HEAD_INIT(test_info.channels),
215 /* poor man's completion - we want to use wait_event_freezable() on it */
256 list_for_each_entry(dtc, &info->channels, node) { in is_threaded_test_run()
259 list_for_each_entry(thread, &dtc->threads, node) { in is_threaded_test_run()
260 if (!thread->done && !thread->pending) in is_threaded_test_run()
272 list_for_each_entry(dtc, &info->channels, node) { in is_threaded_test_pending()
275 list_for_each_entry(thread, &dtc->threads, node) { in is_threaded_test_pending()
276 if (thread->pending) in is_threaded_test_pending()
287 struct dmatest_params *params = &info->params; in dmatest_wait_get()
289 if (params->iterations) in dmatest_wait_get()
305 if (params->channel[0] == '\0') in dmatest_match_channel()
307 return strcmp(dma_chan_name(chan), params->channel) == 0; in dmatest_match_channel()
313 if (params->device[0] == '\0') in dmatest_match_device()
315 return strcmp(dev_name(device->dev), params->device) == 0; in dmatest_match_device()
380 u8 diff = actual ^ pattern; in dmatest_mismatch() local
382 const char *thread_name = current->comm; in dmatest_mismatch()
388 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) in dmatest_mismatch()
391 else if (diff & PATTERN_SRC) in dmatest_mismatch()
428 current->comm, error_count - MAX_ERROR_COUNT); in dmatest_verify()
439 if (!thread->done) { in dmatest_callback()
440 done->done = true; in dmatest_callback()
441 wake_up_all(done->wait); in dmatest_callback()
444 * If thread->done, it means that this callback occurred in dmatest_callback()
458 return val % 2 ? val : val - 1; in min_odd()
466 current->comm, n, err, src_off, dst_off, len, data); in result()
469 current->comm, n, err, src_off, dst_off, len, data); in result()
478 current->comm, n, err, src_off, dst_off, len, data); in dbg_result()
495 /* drop precision until runtime is 32-bits */ in dmatest_persec()
518 kfree(d->raw[i]); in __dmatest_free_test_data()
520 kfree(d->aligned); in __dmatest_free_test_data()
521 kfree(d->raw); in __dmatest_free_test_data()
526 __dmatest_free_test_data(d, d->cnt); in dmatest_free_test_data()
534 d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL); in dmatest_alloc_test_data()
535 if (!d->raw) in dmatest_alloc_test_data()
536 return -ENOMEM; in dmatest_alloc_test_data()
538 d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL); in dmatest_alloc_test_data()
539 if (!d->aligned) in dmatest_alloc_test_data()
542 for (i = 0; i < d->cnt; i++) { in dmatest_alloc_test_data()
543 d->raw[i] = kmalloc(buf_size + align, d->gfp_flags); in dmatest_alloc_test_data()
544 if (!d->raw[i]) in dmatest_alloc_test_data()
549 d->aligned[i] = PTR_ALIGN(d->raw[i], align); in dmatest_alloc_test_data()
551 d->aligned[i] = d->raw[i]; in dmatest_alloc_test_data()
557 return -ENOMEM; in dmatest_alloc_test_data()
564 * in parallel for a single channel, and there may be multiple channels
577 struct dmatest_done *done = &thread->test_done; in dmatest_func()
595 ktime_t ktime, start, diff; in dmatest_func() local
608 ret = -ENOMEM; in dmatest_func()
611 thread->pending = false; in dmatest_func()
612 info = thread->info; in dmatest_func()
613 params = &info->params; in dmatest_func()
614 chan = thread->chan; in dmatest_func()
615 dev = chan->device; in dmatest_func()
618 src = &thread->src; in dmatest_func()
619 dst = &thread->dst; in dmatest_func()
620 if (thread->type == DMA_MEMCPY) { in dmatest_func()
621 align = params->alignment < 0 ? dev->copy_align : in dmatest_func()
622 params->alignment; in dmatest_func()
623 src->cnt = dst->cnt = 1; in dmatest_func()
624 } else if (thread->type == DMA_MEMSET) { in dmatest_func()
625 align = params->alignment < 0 ? dev->fill_align : in dmatest_func()
626 params->alignment; in dmatest_func()
627 src->cnt = dst->cnt = 1; in dmatest_func()
629 } else if (thread->type == DMA_XOR) { in dmatest_func()
631 src->cnt = min_odd(params->xor_sources | 1, dev->max_xor); in dmatest_func()
632 dst->cnt = 1; in dmatest_func()
633 align = params->alignment < 0 ? dev->xor_align : in dmatest_func()
634 params->alignment; in dmatest_func()
635 } else if (thread->type == DMA_PQ) { in dmatest_func()
637 src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0)); in dmatest_func()
638 dst->cnt = 2; in dmatest_func()
639 align = params->alignment < 0 ? dev->pq_align : in dmatest_func()
640 params->alignment; in dmatest_func()
642 pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL); in dmatest_func()
646 for (i = 0; i < src->cnt; i++) in dmatest_func()
652 if ((src->cnt + dst->cnt) >= 255) { in dmatest_func()
654 src->cnt + dst->cnt); in dmatest_func()
658 buf_size = params->buf_size; in dmatest_func()
660 pr_err("%u-byte buffer too small for %d-byte alignment\n", in dmatest_func()
665 src->gfp_flags = GFP_KERNEL; in dmatest_func()
666 dst->gfp_flags = GFP_KERNEL; in dmatest_func()
667 if (params->nobounce) { in dmatest_func()
668 src->gfp_flags = GFP_DMA; in dmatest_func()
669 dst->gfp_flags = GFP_DMA; in dmatest_func()
680 srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL); in dmatest_func()
684 dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL); in dmatest_func()
691 if (params->polled) in dmatest_func()
698 (params->iterations && total_tests >= params->iterations))) { in dmatest_func()
706 if (params->transfer_size) { in dmatest_func()
707 if (params->transfer_size >= buf_size) { in dmatest_func()
708 pr_err("%u-byte transfer size must be lower than %u-buffer size\n", in dmatest_func()
709 params->transfer_size, buf_size); in dmatest_func()
712 len = params->transfer_size; in dmatest_func()
713 } else if (params->norandom) { in dmatest_func()
720 if (!params->transfer_size) { in dmatest_func()
727 if (params->norandom) { in dmatest_func()
728 src->off = 0; in dmatest_func()
729 dst->off = 0; in dmatest_func()
731 src->off = dmatest_random() % (buf_size - len + 1); in dmatest_func()
732 dst->off = dmatest_random() % (buf_size - len + 1); in dmatest_func()
734 src->off = (src->off >> align) << align; in dmatest_func()
735 dst->off = (dst->off >> align) << align; in dmatest_func()
738 if (!params->noverify) { in dmatest_func()
740 dmatest_init_srcs(src->aligned, src->off, len, in dmatest_func()
742 dmatest_init_dsts(dst->aligned, dst->off, len, in dmatest_func()
745 diff = ktime_sub(ktime_get(), start); in dmatest_func()
746 filltime = ktime_add(filltime, diff); in dmatest_func()
749 um = dmaengine_get_unmap_data(dma_dev, src->cnt + dst->cnt, in dmatest_func()
754 src->off, dst->off, len, ret); in dmatest_func()
758 um->len = buf_size; in dmatest_func()
759 for (i = 0; i < src->cnt; i++) { in dmatest_func()
760 void *buf = src->aligned[i]; in dmatest_func()
764 um->addr[i] = dma_map_page(dma_dev, pg, pg_off, in dmatest_func()
765 um->len, DMA_TO_DEVICE); in dmatest_func()
766 srcs[i] = um->addr[i] + src->off; in dmatest_func()
767 ret = dma_mapping_error(dma_dev, um->addr[i]); in dmatest_func()
770 src->off, dst->off, len, ret); in dmatest_func()
773 um->to_cnt++; in dmatest_func()
776 dsts = &um->addr[src->cnt]; in dmatest_func()
777 for (i = 0; i < dst->cnt; i++) { in dmatest_func()
778 void *buf = dst->aligned[i]; in dmatest_func()
782 dsts[i] = dma_map_page(dma_dev, pg, pg_off, um->len, in dmatest_func()
787 src->off, dst->off, len, ret); in dmatest_func()
790 um->bidi_cnt++; in dmatest_func()
793 if (thread->type == DMA_MEMCPY) in dmatest_func()
794 tx = dev->device_prep_dma_memcpy(chan, in dmatest_func()
795 dsts[0] + dst->off, in dmatest_func()
797 else if (thread->type == DMA_MEMSET) in dmatest_func()
798 tx = dev->device_prep_dma_memset(chan, in dmatest_func()
799 dsts[0] + dst->off, in dmatest_func()
800 *(src->aligned[0] + src->off), in dmatest_func()
802 else if (thread->type == DMA_XOR) in dmatest_func()
803 tx = dev->device_prep_dma_xor(chan, in dmatest_func()
804 dsts[0] + dst->off, in dmatest_func()
805 srcs, src->cnt, in dmatest_func()
807 else if (thread->type == DMA_PQ) { in dmatest_func()
808 for (i = 0; i < dst->cnt; i++) in dmatest_func()
809 dma_pq[i] = dsts[i] + dst->off; in dmatest_func()
810 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs, in dmatest_func()
811 src->cnt, pq_coefs, in dmatest_func()
816 result("prep error", total_tests, src->off, in dmatest_func()
817 dst->off, len, ret); in dmatest_func()
822 done->done = false; in dmatest_func()
823 if (!params->polled) { in dmatest_func()
824 tx->callback = dmatest_callback; in dmatest_func()
825 tx->callback_param = done; in dmatest_func()
827 cookie = tx->tx_submit(tx); in dmatest_func()
830 result("submit error", total_tests, src->off, in dmatest_func()
831 dst->off, len, ret); in dmatest_func()
836 if (params->polled) { in dmatest_func()
840 done->done = true; in dmatest_func()
844 wait_event_freezable_timeout(thread->done_wait, in dmatest_func()
845 done->done, in dmatest_func()
846 msecs_to_jiffies(params->timeout)); in dmatest_func()
852 if (!done->done) { in dmatest_func()
853 result("test timed out", total_tests, src->off, dst->off, in dmatest_func()
858 dev->cap_mask) && in dmatest_func()
862 "completion busy status", total_tests, src->off, in dmatest_func()
863 dst->off, len, ret); in dmatest_func()
869 if (params->noverify) { in dmatest_func()
870 verbose_result("test passed", total_tests, src->off, in dmatest_func()
871 dst->off, len, 0); in dmatest_func()
876 pr_debug("%s: verifying source buffer...\n", current->comm); in dmatest_func()
877 error_count = dmatest_verify(src->aligned, 0, src->off, in dmatest_func()
879 error_count += dmatest_verify(src->aligned, src->off, in dmatest_func()
880 src->off + len, src->off, in dmatest_func()
882 error_count += dmatest_verify(src->aligned, src->off + len, in dmatest_func()
883 buf_size, src->off + len, in dmatest_func()
886 pr_debug("%s: verifying dest buffer...\n", current->comm); in dmatest_func()
887 error_count += dmatest_verify(dst->aligned, 0, dst->off, in dmatest_func()
890 error_count += dmatest_verify(dst->aligned, dst->off, in dmatest_func()
891 dst->off + len, src->off, in dmatest_func()
894 error_count += dmatest_verify(dst->aligned, dst->off + len, in dmatest_func()
895 buf_size, dst->off + len, in dmatest_func()
898 diff = ktime_sub(ktime_get(), start); in dmatest_func()
899 comparetime = ktime_add(comparetime, diff); in dmatest_func()
902 result("data error", total_tests, src->off, dst->off, in dmatest_func()
906 verbose_result("test passed", total_tests, src->off, in dmatest_func()
907 dst->off, len, 0); in dmatest_func()
934 current->comm, total_tests, failed_tests, in dmatest_func()
938 /* terminate all transfers on specified channels */ in dmatest_func()
942 thread->done = true; in dmatest_func()
954 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { in dmatest_cleanup_channel()
955 ret = kthread_stop(thread->task); in dmatest_cleanup_channel()
957 thread->task->comm, ret); in dmatest_cleanup_channel()
958 list_del(&thread->node); in dmatest_cleanup_channel()
959 put_task_struct(thread->task); in dmatest_cleanup_channel()
963 /* terminate all transfers on specified channels */ in dmatest_cleanup_channel()
964 dmaengine_terminate_sync(dtc->chan); in dmatest_cleanup_channel()
972 struct dmatest_params *params = &info->params; in dmatest_add_threads()
974 struct dma_chan *chan = dtc->chan; in dmatest_add_threads()
987 return -EINVAL; in dmatest_add_threads()
989 for (i = 0; i < params->threads_per_chan; i++) { in dmatest_add_threads()
992 pr_warn("No memory for %s-%s%u\n", in dmatest_add_threads()
996 thread->info = info; in dmatest_add_threads()
997 thread->chan = dtc->chan; in dmatest_add_threads()
998 thread->type = type; in dmatest_add_threads()
999 thread->test_done.wait = &thread->done_wait; in dmatest_add_threads()
1000 init_waitqueue_head(&thread->done_wait); in dmatest_add_threads()
1002 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u", in dmatest_add_threads()
1004 if (IS_ERR(thread->task)) { in dmatest_add_threads()
1005 pr_warn("Failed to create thread %s-%s%u\n", in dmatest_add_threads()
1012 get_task_struct(thread->task); in dmatest_add_threads()
1013 list_add_tail(&thread->node, &dtc->threads); in dmatest_add_threads()
1014 thread->pending = true; in dmatest_add_threads()
1024 struct dma_device *dma_dev = chan->device; in dmatest_add_channel()
1031 return -ENOMEM; in dmatest_add_channel()
1034 dtc->chan = chan; in dmatest_add_channel()
1035 INIT_LIST_HEAD(&dtc->threads); in dmatest_add_channel()
1037 if (dma_has_cap(DMA_COMPLETION_NO_ORDER, dma_dev->cap_mask) && in dmatest_add_channel()
1038 info->params.polled) { in dmatest_add_channel()
1039 info->params.polled = false; in dmatest_add_channel()
1043 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { in dmatest_add_channel()
1050 if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) { in dmatest_add_channel()
1057 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { in dmatest_add_channel()
1061 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { in dmatest_add_channel()
1069 list_add_tail(&dtc->node, &info->channels); in dmatest_add_channel()
1070 info->nr_channels++; in dmatest_add_channel()
1077 return dmatest_match_channel(param, chan) && dmatest_match_device(param, chan->device); in filter()
1088 struct dmatest_params *params = &info->params; in request_channels()
1098 break; /* no more channels available */ in request_channels()
1099 if (params->max_channels && in request_channels()
1100 info->nr_channels >= params->max_channels) in request_channels()
1107 struct dmatest_params *params = &info->params; in add_threaded_test()
1110 params->nobounce = nobounce; in add_threaded_test()
1111 params->buf_size = test_buf_size; in add_threaded_test()
1112 strscpy(params->channel, strim(test_channel), sizeof(params->channel)); in add_threaded_test()
1113 strscpy(params->device, strim(test_device), sizeof(params->device)); in add_threaded_test()
1114 params->threads_per_chan = threads_per_chan; in add_threaded_test()
1115 params->max_channels = max_channels; in add_threaded_test()
1116 params->iterations = iterations; in add_threaded_test()
1117 params->xor_sources = xor_sources; in add_threaded_test()
1118 params->pq_sources = pq_sources; in add_threaded_test()
1119 params->timeout = timeout; in add_threaded_test()
1120 params->noverify = noverify; in add_threaded_test()
1121 params->norandom = norandom; in add_threaded_test()
1122 params->alignment = alignment; in add_threaded_test()
1123 params->transfer_size = transfer_size; in add_threaded_test()
1124 params->polled = polled; in add_threaded_test()
1137 list_for_each_entry(dtc, &info->channels, node) { in run_pending_tests()
1141 list_for_each_entry(thread, &dtc->threads, node) { in run_pending_tests()
1142 wake_up_process(thread->task); in run_pending_tests()
1146 thread_count, dma_chan_name(dtc->chan)); in run_pending_tests()
1155 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) { in stop_threaded_test()
1156 list_del(&dtc->node); in stop_threaded_test()
1157 chan = dtc->chan; in stop_threaded_test()
1163 info->nr_channels = 0; in stop_threaded_test()
1171 if (!info->did_init) in start_threaded_tests()
1181 mutex_lock(&info->lock); in dmatest_run_get()
1189 mutex_unlock(&info->lock); in dmatest_run_get()
1199 mutex_lock(&info->lock); in dmatest_run_set()
1202 mutex_unlock(&info->lock); in dmatest_run_set()
1209 ret = info->last_error; in dmatest_run_set()
1213 mutex_unlock(&info->lock); in dmatest_run_set()
1217 pr_info("No channels configured, continue with any\n"); in dmatest_run_set()
1228 mutex_unlock(&info->lock); in dmatest_run_set()
1240 mutex_lock(&info->lock); in dmatest_chan_set()
1243 mutex_unlock(&info->lock); in dmatest_chan_set()
1249 /* Reject channels that are already registered */ in dmatest_chan_set()
1251 list_for_each_entry(dtc, &info->channels, node) { in dmatest_chan_set()
1252 if (strcmp(dma_chan_name(dtc->chan), in dmatest_chan_set()
1254 dtc = list_last_entry(&info->channels, in dmatest_chan_set()
1258 dma_chan_name(dtc->chan), in dmatest_chan_set()
1260 ret = -EBUSY; in dmatest_chan_set()
1269 if (!list_empty(&info->channels)) { in dmatest_chan_set()
1276 dtc = list_last_entry(&info->channels, struct dmatest_chan, node); in dmatest_chan_set()
1277 if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0) in dmatest_chan_set()
1279 ret = -EINVAL; in dmatest_chan_set()
1280 strscpy(chan_reset_val, dma_chan_name(dtc->chan), in dmatest_chan_set()
1286 /* Clear test_channel if no channels were added successfully */ in dmatest_chan_set()
1288 ret = -EBUSY; in dmatest_chan_set()
1292 info->last_error = ret; in dmatest_chan_set()
1293 mutex_unlock(&info->lock); in dmatest_chan_set()
1299 info->last_error = ret; in dmatest_chan_set()
1300 mutex_unlock(&info->lock); in dmatest_chan_set()
1309 mutex_lock(&info->lock); in dmatest_chan_get()
1314 mutex_unlock(&info->lock); in dmatest_chan_get()
1325 list_for_each_entry(dtc, &info->channels, node) { in dmatest_test_list_get()
1329 list_for_each_entry(thread, &dtc->threads, node) { in dmatest_test_list_get()
1333 thread_count, dma_chan_name(dtc->chan)); in dmatest_test_list_get()
1342 struct dmatest_params *params = &info->params; in dmatest_init()
1345 mutex_lock(&info->lock); in dmatest_init()
1348 mutex_unlock(&info->lock); in dmatest_init()
1351 if (params->iterations && wait) in dmatest_init()
1357 info->did_init = true; in dmatest_init()
1361 /* when compiled-in wait for drivers to load first */
1368 mutex_lock(&info->lock); in dmatest_exit()
1370 mutex_unlock(&info->lock); in dmatest_exit()