Lines Matching +full:t +full:- +full:head
1 // SPDX-License-Identifier: GPL-2.0
3 * Performance events ring-buffer code:
6 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
22 atomic_set(&handle->rb->poll, EPOLLIN); in perf_output_wakeup()
24 handle->event->pending_wakeup = 1; in perf_output_wakeup()
26 if (*perf_event_fasync(handle->event) && !handle->event->pending_kill) in perf_output_wakeup()
27 handle->event->pending_kill = POLL_IN; in perf_output_wakeup()
29 irq_work_queue(&handle->event->pending_irq); in perf_output_wakeup()
33 * We need to ensure a later event_id doesn't publish a head when a former
34 * event isn't done writing. However since we need to deal with NMIs we
37 * We only publish the head (and generate a wakeup) when the outer-most
42 struct perf_buffer *rb = handle->rb; in perf_output_get_handle()
50 (*(volatile unsigned int *)&rb->nest)++; in perf_output_get_handle()
51 handle->wakeup = local_read(&rb->wakeup); in perf_output_get_handle()
56 struct perf_buffer *rb = handle->rb; in perf_output_put_handle()
57 unsigned long head; in perf_output_put_handle() local
61 * If this isn't the outermost nesting, we don't have to update in perf_output_put_handle()
62 * @rb->user_page->data_head. in perf_output_put_handle()
64 nest = READ_ONCE(rb->nest); in perf_output_put_handle()
66 WRITE_ONCE(rb->nest, nest - 1); in perf_output_put_handle()
72 * In order to avoid publishing a head value that goes backwards, in perf_output_put_handle()
73 * we must ensure the load of @rb->head happens after we've in perf_output_put_handle()
74 * incremented @rb->nest. in perf_output_put_handle()
76 * Otherwise we can observe a @rb->head value before one published in perf_output_put_handle()
80 head = local_read(&rb->head); in perf_output_put_handle()
83 * IRQ/NMI can happen here and advance @rb->head, causing our in perf_output_put_handle()
92 * if (LOAD ->data_tail) { LOAD ->data_head in perf_output_put_handle()
96 * STORE ->data_head STORE ->data_tail in perf_output_put_handle()
102 * the ->data_tail and the stores of $data. In case ->data_tail in perf_output_put_handle()
114 WRITE_ONCE(rb->user_page->data_head, head); in perf_output_put_handle()
117 * We must publish the head before decrementing the nest count, in perf_output_put_handle()
118 * otherwise an IRQ/NMI can publish a more recent head value and our in perf_output_put_handle()
122 WRITE_ONCE(rb->nest, 0); in perf_output_put_handle()
125 * Ensure we decrement @rb->nest before we validate the @rb->head. in perf_output_put_handle()
129 if (unlikely(head != local_read(&rb->head))) { in perf_output_put_handle()
130 WRITE_ONCE(rb->nest, 1); in perf_output_put_handle()
134 if (handle->wakeup != local_read(&rb->wakeup)) in perf_output_put_handle()
142 ring_buffer_has_space(unsigned long head, unsigned long tail, in ring_buffer_has_space() argument
147 return CIRC_SPACE(head, tail, data_size) >= size; in ring_buffer_has_space()
149 return CIRC_SPACE(tail, head, data_size) >= size; in ring_buffer_has_space()
159 unsigned long tail, offset, head; in __perf_output_begin() local
171 if (event->parent) in __perf_output_begin()
172 event = event->parent; in __perf_output_begin()
174 rb = rcu_dereference(event->rb); in __perf_output_begin()
178 if (unlikely(rb->paused)) { in __perf_output_begin()
179 if (rb->nr_pages) { in __perf_output_begin()
180 local_inc(&rb->lost); in __perf_output_begin()
181 atomic64_inc(&event->lost_samples); in __perf_output_begin()
186 handle->rb = rb; in __perf_output_begin()
187 handle->event = event; in __perf_output_begin()
189 have_lost = local_read(&rb->lost); in __perf_output_begin()
192 if (event->attr.sample_id_all) in __perf_output_begin()
193 size += event->id_header_size; in __perf_output_begin()
198 offset = local_read(&rb->head); in __perf_output_begin()
200 head = offset; in __perf_output_begin()
201 tail = READ_ONCE(rb->user_page->data_tail); in __perf_output_begin()
202 if (!rb->overwrite) { in __perf_output_begin()
203 if (unlikely(!ring_buffer_has_space(head, tail, in __perf_output_begin()
222 head += size; in __perf_output_begin()
224 head -= size; in __perf_output_begin()
225 } while (!local_try_cmpxchg(&rb->head, &offset, head)); in __perf_output_begin()
228 offset = head; in __perf_output_begin()
229 head = (u64)(-head); in __perf_output_begin()
237 if (unlikely(head - local_read(&rb->wakeup) > rb->watermark)) in __perf_output_begin()
238 local_add(rb->watermark, &rb->wakeup); in __perf_output_begin()
242 handle->page = (offset >> page_shift) & (rb->nr_pages - 1); in __perf_output_begin()
243 offset &= (1UL << page_shift) - 1; in __perf_output_begin()
244 handle->addr = rb->data_pages[handle->page] + offset; in __perf_output_begin()
245 handle->size = (1UL << page_shift) - offset; in __perf_output_begin()
251 lost_event.id = event->id; in __perf_output_begin()
252 lost_event.lost = local_xchg(&rb->lost, 0); in __perf_output_begin()
263 local_inc(&rb->lost); in __perf_output_begin()
264 atomic64_inc(&event->lost_samples); in __perf_output_begin()
269 return -ENOSPC; in __perf_output_begin()
319 rb->watermark = min(max_size, watermark); in ring_buffer_init()
321 if (!rb->watermark) in ring_buffer_init()
322 rb->watermark = max_size / 2; in ring_buffer_init()
325 rb->overwrite = 0; in ring_buffer_init()
327 rb->overwrite = 1; in ring_buffer_init()
329 refcount_set(&rb->refcount, 1); in ring_buffer_init()
331 INIT_LIST_HEAD(&rb->event_list); in ring_buffer_init()
332 spin_lock_init(&rb->event_lock); in ring_buffer_init()
335 * perf_output_begin() only checks rb->paused, therefore in ring_buffer_init()
336 * rb->paused must be true if we have no pages for output. in ring_buffer_init()
338 if (!rb->nr_pages) in ring_buffer_init()
339 rb->paused = 1; in ring_buffer_init()
341 mutex_init(&rb->aux_mutex); in ring_buffer_init()
347 * OVERWRITE is determined by perf_aux_output_end() and can't in perf_aux_output_flag()
353 handle->aux_flags |= flags; in perf_aux_output_flag()
379 if (output_event->parent) in perf_aux_output_begin()
380 output_event = output_event->parent; in perf_aux_output_begin()
385 * to make sure it doesn't disappear under us. in perf_aux_output_begin()
402 if (!atomic_read(&rb->aux_mmap_count)) in perf_aux_output_begin()
405 if (!refcount_inc_not_zero(&rb->aux_refcount)) in perf_aux_output_begin()
408 nest = READ_ONCE(rb->aux_nest); in perf_aux_output_begin()
416 WRITE_ONCE(rb->aux_nest, nest + 1); in perf_aux_output_begin()
418 aux_head = rb->aux_head; in perf_aux_output_begin()
420 handle->rb = rb; in perf_aux_output_begin()
421 handle->event = event; in perf_aux_output_begin()
422 handle->head = aux_head; in perf_aux_output_begin()
423 handle->size = 0; in perf_aux_output_begin()
424 handle->aux_flags = 0; in perf_aux_output_begin()
429 * (B) <-> (C) ordering is still observed by the pmu driver. in perf_aux_output_begin()
431 if (!rb->aux_overwrite) { in perf_aux_output_begin()
432 aux_tail = READ_ONCE(rb->user_page->aux_tail); in perf_aux_output_begin()
433 handle->wakeup = rb->aux_wakeup + rb->aux_watermark; in perf_aux_output_begin()
434 if (aux_head - aux_tail < perf_aux_size(rb)) in perf_aux_output_begin()
435 handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb)); in perf_aux_output_begin()
438 * handle->size computation depends on aux_tail load; this forms a in perf_aux_output_begin()
442 if (!handle->size) { /* A, matches D */ in perf_aux_output_begin()
443 event->pending_disable = smp_processor_id(); in perf_aux_output_begin()
445 WRITE_ONCE(rb->aux_nest, 0); in perf_aux_output_begin()
450 return handle->rb->aux_priv; in perf_aux_output_begin()
453 /* can't be last */ in perf_aux_output_begin()
458 handle->event = NULL; in perf_aux_output_begin()
466 if (rb->aux_overwrite) in rb_need_aux_wakeup()
469 if (rb->aux_head - rb->aux_wakeup >= rb->aux_watermark) { in rb_need_aux_wakeup()
470 rb->aux_wakeup = rounddown(rb->aux_head, rb->aux_watermark); in rb_need_aux_wakeup()
489 bool wakeup = !!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED); in perf_aux_output_end()
490 struct perf_buffer *rb = handle->rb; in perf_aux_output_end()
494 if (rb->aux_overwrite) { in perf_aux_output_end()
495 handle->aux_flags |= PERF_AUX_FLAG_OVERWRITE; in perf_aux_output_end()
497 aux_head = handle->head; in perf_aux_output_end()
498 rb->aux_head = aux_head; in perf_aux_output_end()
500 handle->aux_flags &= ~PERF_AUX_FLAG_OVERWRITE; in perf_aux_output_end()
502 aux_head = rb->aux_head; in perf_aux_output_end()
503 rb->aux_head += size; in perf_aux_output_end()
510 * useful, as they don't communicate any *new* information, in perf_aux_output_end()
511 * aside from the short-lived offset, that becomes history at in perf_aux_output_end()
512 * the next event sched-in and therefore isn't useful. in perf_aux_output_end()
515 * offset. So, from now on we don't output AUX records that in perf_aux_output_end()
518 if (size || (handle->aux_flags & ~(u64)PERF_AUX_FLAG_OVERWRITE)) in perf_aux_output_end()
519 perf_event_aux_event(handle->event, aux_head, size, in perf_aux_output_end()
520 handle->aux_flags); in perf_aux_output_end()
522 WRITE_ONCE(rb->user_page->aux_head, rb->aux_head); in perf_aux_output_end()
527 if (handle->aux_flags & PERF_AUX_FLAG_TRUNCATED) in perf_aux_output_end()
528 handle->event->pending_disable = smp_processor_id(); in perf_aux_output_end()
532 handle->event = NULL; in perf_aux_output_end()
534 WRITE_ONCE(rb->aux_nest, 0); in perf_aux_output_end()
535 /* can't be last */ in perf_aux_output_end()
547 struct perf_buffer *rb = handle->rb; in perf_aux_output_skip()
549 if (size > handle->size) in perf_aux_output_skip()
550 return -ENOSPC; in perf_aux_output_skip()
552 rb->aux_head += size; in perf_aux_output_skip()
554 WRITE_ONCE(rb->user_page->aux_head, rb->aux_head); in perf_aux_output_skip()
557 handle->wakeup = rb->aux_wakeup + rb->aux_watermark; in perf_aux_output_skip()
560 handle->head = rb->aux_head; in perf_aux_output_skip()
561 handle->size -= size; in perf_aux_output_skip()
570 if (!handle->event) in perf_get_aux()
573 return handle->rb->aux_priv; in perf_get_aux()
584 struct perf_buffer *rb = aux_handle->rb; in perf_output_copy_aux()
588 from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1; in perf_output_copy_aux()
589 to &= (rb->aux_nr_pages << PAGE_SHIFT) - 1; in perf_output_copy_aux()
592 tocopy = PAGE_SIZE - offset_in_page(from); in perf_output_copy_aux()
594 tocopy = min(tocopy, to - from); in perf_output_copy_aux()
598 addr = rb->aux_pages[from >> PAGE_SHIFT]; in perf_output_copy_aux()
603 return -EFAULT; in perf_output_copy_aux()
607 from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1; in perf_output_copy_aux()
624 } while (!page && order--); in rb_alloc_aux_page()
629 * if we managed to secure a high-order allocation, in rb_alloc_aux_page()
643 struct page *page = virt_to_page(rb->aux_pages[idx]); in rb_free_aux_page()
646 page->mapping = NULL; in rb_free_aux_page()
662 if (rb->aux_priv) { in __rb_free_aux()
663 rb->free_aux(rb->aux_priv); in __rb_free_aux()
664 rb->free_aux = NULL; in __rb_free_aux()
665 rb->aux_priv = NULL; in __rb_free_aux()
668 if (rb->aux_nr_pages) { in __rb_free_aux()
669 for (pg = 0; pg < rb->aux_nr_pages; pg++) in __rb_free_aux()
672 kfree(rb->aux_pages); in __rb_free_aux()
673 rb->aux_nr_pages = 0; in __rb_free_aux()
681 int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu); in rb_alloc_aux()
682 int ret = -ENOMEM, max_order; in rb_alloc_aux()
685 return -EOPNOTSUPP; in rb_alloc_aux()
688 return -EINVAL; in rb_alloc_aux()
698 (unsigned long)nr_pages << (PAGE_SHIFT - 1)); in rb_alloc_aux()
719 return -ENOMEM; in rb_alloc_aux()
720 rb->aux_pages = kcalloc_node(nr_pages, sizeof(void *), GFP_KERNEL, in rb_alloc_aux()
722 if (!rb->aux_pages) in rb_alloc_aux()
723 return -ENOMEM; in rb_alloc_aux()
725 rb->free_aux = event->pmu->free_aux; in rb_alloc_aux()
726 for (rb->aux_nr_pages = 0; rb->aux_nr_pages < nr_pages;) { in rb_alloc_aux()
730 order = min(max_order, ilog2(nr_pages - rb->aux_nr_pages)); in rb_alloc_aux()
735 for (last = rb->aux_nr_pages + (1 << page_private(page)); in rb_alloc_aux()
736 last > rb->aux_nr_pages; rb->aux_nr_pages++) in rb_alloc_aux()
737 rb->aux_pages[rb->aux_nr_pages] = page_address(page++); in rb_alloc_aux()
741 * In overwrite mode, PMUs that don't support SG may not handle more in rb_alloc_aux()
746 if ((event->pmu->capabilities & PERF_PMU_CAP_AUX_NO_SG) && in rb_alloc_aux()
748 struct page *page = virt_to_page(rb->aux_pages[0]); in rb_alloc_aux()
754 rb->aux_priv = event->pmu->setup_aux(event, rb->aux_pages, nr_pages, in rb_alloc_aux()
756 if (!rb->aux_priv) in rb_alloc_aux()
767 refcount_set(&rb->aux_refcount, 1); in rb_alloc_aux()
769 rb->aux_overwrite = overwrite; in rb_alloc_aux()
770 rb->aux_watermark = watermark; in rb_alloc_aux()
774 rb->aux_pgoff = pgoff; in rb_alloc_aux()
783 if (refcount_dec_and_test(&rb->aux_refcount)) in rb_free_aux()
790 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
796 if (pgoff > rb->nr_pages) in __perf_mmap_to_page()
800 return virt_to_page(rb->user_page); in __perf_mmap_to_page()
802 return virt_to_page(rb->data_pages[pgoff - 1]); in __perf_mmap_to_page()
810 node = (cpu == -1) ? cpu : cpu_to_node(cpu); in perf_mmap_alloc_page()
822 page->mapping = NULL; in perf_mmap_free_page()
838 node = (cpu == -1) ? cpu : cpu_to_node(cpu); in rb_alloc()
843 rb->user_page = perf_mmap_alloc_page(cpu); in rb_alloc()
844 if (!rb->user_page) in rb_alloc()
848 rb->data_pages[i] = perf_mmap_alloc_page(cpu); in rb_alloc()
849 if (!rb->data_pages[i]) in rb_alloc()
853 rb->nr_pages = nr_pages; in rb_alloc()
860 for (i--; i >= 0; i--) in rb_alloc()
861 perf_mmap_free_page(rb->data_pages[i]); in rb_alloc()
863 perf_mmap_free_page(rb->user_page); in rb_alloc()
876 perf_mmap_free_page(rb->user_page); in rb_free()
877 for (i = 0; i < rb->nr_pages; i++) in rb_free()
878 perf_mmap_free_page(rb->data_pages[i]); in rb_free()
890 return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE); in __perf_mmap_to_page()
897 page->mapping = NULL; in perf_mmap_unmark_page()
909 base = rb->user_page; in rb_free_work()
920 schedule_work(&rb->work); in rb_free()
933 node = (cpu == -1) ? cpu : cpu_to_node(cpu); in rb_alloc()
938 INIT_WORK(&rb->work, rb_free_work); in rb_alloc()
944 rb->user_page = all_buf; in rb_alloc()
945 rb->data_pages[0] = all_buf + PAGE_SIZE; in rb_alloc()
947 rb->nr_pages = 1; in rb_alloc()
948 rb->page_order = ilog2(nr_pages); in rb_alloc()
967 if (rb->aux_nr_pages) { in perf_mmap_to_page()
969 if (pgoff > rb->aux_pgoff + rb->aux_nr_pages) in perf_mmap_to_page()
973 if (pgoff >= rb->aux_pgoff) { in perf_mmap_to_page()
974 int aux_pgoff = array_index_nospec(pgoff - rb->aux_pgoff, rb->aux_nr_pages); in perf_mmap_to_page()
975 return virt_to_page(rb->aux_pages[aux_pgoff]); in perf_mmap_to_page()