1 // SPDX-License-Identifier: GPL-2.0
19 	int ringbuf_sz; /* per-ringbuf, in bytes */
21 	int perfbuf_sz; /* per-CPU size, in pages */
41 	{ "rb-b2b", ARG_RB_BACK2BACK, NULL, 0, "Back-to-back mode"},
42 …{ "rb-use-output", ARG_RB_USE_OUTPUT, NULL, 0, "Use bpf_ringbuf_output() instead of bpf_ringbuf_re…
43 	{ "rb-batch-cnt", ARG_RB_BATCH_CNT, "CNT", 0, "Set BPF-side record batch count"},
44 	{ "rb-sampled", ARG_RB_SAMPLED, NULL, 0, "Notification sampling"},
45 	{ "rb-sample-rate", ARG_RB_SAMPLE_RATE, "RATE", 0, "Notification sample rate"},
87 /* RINGBUF-LIBBPF benchmark */
99 		fprintf(stderr, "rb-libbpf benchmark needs one consumer!\n");  in bufs_validate()
104 		fprintf(stderr, "back-to-back mode makes sense only for single-producer case!\n");  in bufs_validate()
131 	res->hits = atomic_swap(&buf_hits.value, 0);  in ringbuf_libbpf_measure()
132 	res->drops = atomic_swap(&ctx->skel->bss->dropped, 0);  in ringbuf_libbpf_measure()
147 	skel->rodata->batch_cnt = args.batch_cnt;  in ringbuf_setup_skeleton()
148 	skel->rodata->use_output = args.ringbuf_use_output ? 1 : 0;  in ringbuf_setup_skeleton()
152 		skel->rodata->wakeup_data_size = args.sample_rate * 16;  in ringbuf_setup_skeleton()
154 	bpf_map__set_max_entries(skel->maps.ringbuf, args.ringbuf_sz);  in ringbuf_setup_skeleton()
175 	ctx->skel = ringbuf_setup_skeleton();  in ringbuf_libbpf_setup()
176 	ctx->ringbuf = ring_buffer__new(bpf_map__fd(ctx->skel->maps.ringbuf),  in ringbuf_libbpf_setup()
178 	if (!ctx->ringbuf) {  in ringbuf_libbpf_setup()
183 	link = bpf_program__attach(ctx->skel->progs.bench_ringbuf);  in ringbuf_libbpf_setup()
194 	while (ring_buffer__poll(ctx->ringbuf, -1) >= 0) {  in ringbuf_libbpf_consumer()
202 /* RINGBUF-CUSTOM benchmark */
222 	res->hits = atomic_swap(&buf_hits.value, 0);  in ringbuf_custom_measure()
223 	res->drops = atomic_swap(&ctx->skel->bss->dropped, 0);  in ringbuf_custom_measure()
235 	ctx->skel = ringbuf_setup_skeleton();  in ringbuf_custom_setup()
237 	ctx->epoll_fd = epoll_create1(EPOLL_CLOEXEC);  in ringbuf_custom_setup()
238 	if (ctx->epoll_fd < 0) {  in ringbuf_custom_setup()
239 		fprintf(stderr, "failed to create epoll fd: %d\n", -errno);  in ringbuf_custom_setup()
243 	r = &ctx->ringbuf;  in ringbuf_custom_setup()
244 	r->map_fd = bpf_map__fd(ctx->skel->maps.ringbuf);  in ringbuf_custom_setup()
245 	r->mask = args.ringbuf_sz - 1;  in ringbuf_custom_setup()
249 		   r->map_fd, 0);  in ringbuf_custom_setup()
251 		fprintf(stderr, "failed to mmap consumer page: %d\n", -errno);  in ringbuf_custom_setup()
254 	r->consumer_pos = tmp;  in ringbuf_custom_setup()
256 	/* Map read-only producer page and data pages. */  in ringbuf_custom_setup()
258 		   r->map_fd, page_size);  in ringbuf_custom_setup()
260 		fprintf(stderr, "failed to mmap data pages: %d\n", -errno);  in ringbuf_custom_setup()
263 	r->producer_pos = tmp;  in ringbuf_custom_setup()
264 	r->data = tmp + page_size;  in ringbuf_custom_setup()
266 	ctx->event.events = EPOLLIN;  in ringbuf_custom_setup()
267 	err = epoll_ctl(ctx->epoll_fd, EPOLL_CTL_ADD, r->map_fd, &ctx->event);  in ringbuf_custom_setup()
269 		fprintf(stderr, "failed to epoll add ringbuf: %d\n", -errno);  in ringbuf_custom_setup()
273 	link = bpf_program__attach(ctx->skel->progs.bench_ringbuf);  in ringbuf_custom_setup()
301 	cons_pos = smp_load_acquire(r->consumer_pos);  in ringbuf_custom_process_ring()
304 		prod_pos = smp_load_acquire(r->producer_pos);  in ringbuf_custom_process_ring()
306 			len_ptr = r->data + (cons_pos & r->mask);  in ringbuf_custom_process_ring()
319 			smp_store_release(r->consumer_pos, cons_pos);  in ringbuf_custom_process_ring()
328 	int cnt;  in ringbuf_custom_consumer()  local
333 		cnt = epoll_wait(ctx->epoll_fd, &ctx->event, 1, -1);  in ringbuf_custom_consumer()
334 		if (cnt > 0)  in ringbuf_custom_consumer()
335 			ringbuf_custom_process_ring(&ctx->ringbuf);  in ringbuf_custom_consumer()
336 	} while (cnt >= 0);  in ringbuf_custom_consumer()
341 /* PERFBUF-LIBBPF benchmark */
351 	res->hits = atomic_swap(&buf_hits.value, 0);  in perfbuf_measure()
352 	res->drops = atomic_swap(&ctx->skel->bss->dropped, 0);  in perfbuf_measure()
367 	skel->rodata->batch_cnt = args.batch_cnt;  in perfbuf_setup_skeleton()
381 	switch (e->type) {  in perfbuf_process_sample_raw()
396 	struct perf_event_attr attr;  in perfbuf_libbpf_setup()  local
399 	ctx->skel = perfbuf_setup_skeleton();  in perfbuf_libbpf_setup()
401 	memset(&attr, 0, sizeof(attr));  in perfbuf_libbpf_setup()
402 	attr.config = PERF_COUNT_SW_BPF_OUTPUT;  in perfbuf_libbpf_setup()
403 	attr.type = PERF_TYPE_SOFTWARE;  in perfbuf_libbpf_setup()
404 	attr.sample_type = PERF_SAMPLE_RAW;  in perfbuf_libbpf_setup()
407 		attr.sample_period = args.sample_rate;  in perfbuf_libbpf_setup()
408 		attr.wakeup_events = args.sample_rate;  in perfbuf_libbpf_setup()
410 		attr.sample_period = 1;  in perfbuf_libbpf_setup()
411 		attr.wakeup_events = 1;  in perfbuf_libbpf_setup()
420 	ctx->perfbuf = perf_buffer__new_raw(bpf_map__fd(ctx->skel->maps.perfbuf),  in perfbuf_libbpf_setup()
421 					    args.perfbuf_sz, &attr,  in perfbuf_libbpf_setup()
423 	if (!ctx->perfbuf) {  in perfbuf_libbpf_setup()
428 	link = bpf_program__attach(ctx->skel->progs.bench_perfbuf);  in perfbuf_libbpf_setup()
439 	while (perf_buffer__poll(ctx->perfbuf, -1) >= 0) {  in perfbuf_libbpf_consumer()
447 /* PERFBUF-CUSTOM benchmark */
478 	struct perf_buffer *pb = ctx->perfbuf;  in perfbuf_custom_consumer()
481 	size_t mmap_mask = pb->mmap_size - 1;  in perfbuf_custom_consumer()
486 	int i, cnt;  in perfbuf_custom_consumer()  local
491 		cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, -1);  in perfbuf_custom_consumer()
492 		if (cnt <= 0) {  in perfbuf_custom_consumer()
493 			fprintf(stderr, "perf epoll failed: %d\n", -errno);  in perfbuf_custom_consumer()
497 		for (i = 0; i < cnt; ++i) {  in perfbuf_custom_consumer()
498 			cpu_buf = pb->events[i].data.ptr;  in perfbuf_custom_consumer()
499 			header = cpu_buf->base;  in perfbuf_custom_consumer()
500 			base = ((void *)header) + pb->page_size;  in perfbuf_custom_consumer()
503 			data_tail = header->data_tail;  in perfbuf_custom_consumer()
506 				ehdr_size = ehdr->size;  in perfbuf_custom_consumer()
508 				if (ehdr->type == PERF_RECORD_SAMPLE)  in perfbuf_custom_consumer()
520 	.name = "rb-libbpf",
532 	.name = "rb-custom",
544 	.name = "pb-libbpf",
556 	.name = "pb-custom",