22 	atomic_set(&handle->rb->poll, EPOLLIN);  in perf_output_wakeup()
42 	struct perf_buffer *rb = handle->rb;  in perf_output_get_handle()  local
50 	(*(volatile unsigned int *)&rb->nest)++;  in perf_output_get_handle()
51 	handle->wakeup = local_read(&rb->wakeup);  in perf_output_get_handle()
56 	struct perf_buffer *rb = handle->rb;  in perf_output_put_handle()  local
62 	 * @rb->user_page->data_head.  in perf_output_put_handle()
64 	nest = READ_ONCE(rb->nest);  in perf_output_put_handle()
66 		WRITE_ONCE(rb->nest, nest - 1);  in perf_output_put_handle()
73 	 * we must ensure the load of @rb->head happens after we've  in perf_output_put_handle()
74 	 * incremented @rb->nest.  in perf_output_put_handle()
76 	 * Otherwise we can observe a @rb->head value before one published  in perf_output_put_handle()
80 	head = local_read(&rb->head);  in perf_output_put_handle()
83 	 * IRQ/NMI can happen here and advance @rb->head, causing our  in perf_output_put_handle()
114 	WRITE_ONCE(rb->user_page->data_head, head);  in perf_output_put_handle()
122 	WRITE_ONCE(rb->nest, 0);  in perf_output_put_handle()
125 	 * Ensure we decrement @rb->nest before we validate the @rb->head.  in perf_output_put_handle()
129 	if (unlikely(head != local_read(&rb->head))) {  in perf_output_put_handle()
130 		WRITE_ONCE(rb->nest, 1);  in perf_output_put_handle()
134 	if (handle->wakeup != local_read(&rb->wakeup))  in perf_output_put_handle()
158 	struct perf_buffer *rb;  in __perf_output_begin()  local
174 	rb = rcu_dereference(event->rb);  in __perf_output_begin()
175 	if (unlikely(!rb))  in __perf_output_begin()
178 	if (unlikely(rb->paused)) {  in __perf_output_begin()
179 		if (rb->nr_pages) {  in __perf_output_begin()
180 			local_inc(&rb->lost);  in __perf_output_begin()
186 	handle->rb    = rb;  in __perf_output_begin()
189 	have_lost = local_read(&rb->lost);  in __perf_output_begin()
198 	offset = local_read(&rb->head);  in __perf_output_begin()
201 		tail = READ_ONCE(rb->user_page->data_tail);  in __perf_output_begin()
202 		if (!rb->overwrite) {  in __perf_output_begin()
204 							    perf_data_size(rb),  in __perf_output_begin()
225 	} while (!local_try_cmpxchg(&rb->head, &offset, head));  in __perf_output_begin()
237 	if (unlikely(head - local_read(&rb->wakeup) > rb->watermark))  in __perf_output_begin()
238 		local_add(rb->watermark, &rb->wakeup);  in __perf_output_begin()
240 	page_shift = PAGE_SHIFT + page_order(rb);  in __perf_output_begin()
242 	handle->page = (offset >> page_shift) & (rb->nr_pages - 1);  in __perf_output_begin()
244 	handle->addr = rb->data_pages[handle->page] + offset;  in __perf_output_begin()
252 		lost_event.lost        = local_xchg(&rb->lost, 0);  in __perf_output_begin()
263 	local_inc(&rb->lost);  in __perf_output_begin()
314 ring_buffer_init(struct perf_buffer *rb, long watermark, int flags)  in ring_buffer_init()  argument
316 	long max_size = perf_data_size(rb);  in ring_buffer_init()
319 		rb->watermark = min(max_size, watermark);  in ring_buffer_init()
321 	if (!rb->watermark)  in ring_buffer_init()
322 		rb->watermark = max_size / 2;  in ring_buffer_init()
325 		rb->overwrite = 0;  in ring_buffer_init()
327 		rb->overwrite = 1;  in ring_buffer_init()
329 	refcount_set(&rb->refcount, 1);  in ring_buffer_init()
331 	INIT_LIST_HEAD(&rb->event_list);  in ring_buffer_init()
332 	spin_lock_init(&rb->event_lock);  in ring_buffer_init()
335 	 * perf_output_begin() only checks rb->paused, therefore  in ring_buffer_init()
336 	 * rb->paused must be true if we have no pages for output.  in ring_buffer_init()
338 	if (!rb->nr_pages)  in ring_buffer_init()
339 		rb->paused = 1;  in ring_buffer_init()
341 	mutex_init(&rb->aux_mutex);  in ring_buffer_init()
376 	struct perf_buffer *rb;  in perf_aux_output_begin()  local
387 	rb = ring_buffer_get(output_event);  in perf_aux_output_begin()
388 	if (!rb)  in perf_aux_output_begin()
391 	if (!rb_has_aux(rb))  in perf_aux_output_begin()
398 	 * Checking rb::aux_mmap_count and rb::refcount has to be done in  in perf_aux_output_begin()
402 	if (!atomic_read(&rb->aux_mmap_count))  in perf_aux_output_begin()
405 	if (!refcount_inc_not_zero(&rb->aux_refcount))  in perf_aux_output_begin()
408 	nest = READ_ONCE(rb->aux_nest);  in perf_aux_output_begin()
416 	WRITE_ONCE(rb->aux_nest, nest + 1);  in perf_aux_output_begin()
418 	aux_head = rb->aux_head;  in perf_aux_output_begin()
420 	handle->rb = rb;  in perf_aux_output_begin()
431 	if (!rb->aux_overwrite) {  in perf_aux_output_begin()
432 		aux_tail = READ_ONCE(rb->user_page->aux_tail);  in perf_aux_output_begin()
433 		handle->wakeup = rb->aux_wakeup + rb->aux_watermark;  in perf_aux_output_begin()
434 		if (aux_head - aux_tail < perf_aux_size(rb))  in perf_aux_output_begin()
435 			handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));  in perf_aux_output_begin()
445 			WRITE_ONCE(rb->aux_nest, 0);  in perf_aux_output_begin()
450 	return handle->rb->aux_priv;  in perf_aux_output_begin()
454 	rb_free_aux(rb);  in perf_aux_output_begin()
457 	ring_buffer_put(rb);  in perf_aux_output_begin()
464 static __always_inline bool rb_need_aux_wakeup(struct perf_buffer *rb)  in rb_need_aux_wakeup()  argument
466 	if (rb->aux_overwrite)  in rb_need_aux_wakeup()
469 	if (rb->aux_head - rb->aux_wakeup >= rb->aux_watermark) {  in rb_need_aux_wakeup()
470 		rb->aux_wakeup = rounddown(rb->aux_head, rb->aux_watermark);  in rb_need_aux_wakeup()
490 	struct perf_buffer *rb = handle->rb;  in perf_aux_output_end()  local
494 	if (rb->aux_overwrite) {  in perf_aux_output_end()
498 		rb->aux_head = aux_head;  in perf_aux_output_end()
502 		aux_head = rb->aux_head;  in perf_aux_output_end()
503 		rb->aux_head += size;  in perf_aux_output_end()
522 	WRITE_ONCE(rb->user_page->aux_head, rb->aux_head);  in perf_aux_output_end()
523 	if (rb_need_aux_wakeup(rb))  in perf_aux_output_end()
534 	WRITE_ONCE(rb->aux_nest, 0);  in perf_aux_output_end()
536 	rb_free_aux(rb);  in perf_aux_output_end()
537 	ring_buffer_put(rb);  in perf_aux_output_end()
547 	struct perf_buffer *rb = handle->rb;  in perf_aux_output_skip()  local
552 	rb->aux_head += size;  in perf_aux_output_skip()
554 	WRITE_ONCE(rb->user_page->aux_head, rb->aux_head);  in perf_aux_output_skip()
555 	if (rb_need_aux_wakeup(rb)) {  in perf_aux_output_skip()
557 		handle->wakeup = rb->aux_wakeup + rb->aux_watermark;  in perf_aux_output_skip()
560 	handle->head = rb->aux_head;  in perf_aux_output_skip()
573 	return handle->rb->aux_priv;  in perf_get_aux()
584 	struct perf_buffer *rb = aux_handle->rb;  in perf_output_copy_aux()  local
588 	from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;  in perf_output_copy_aux()
589 	to &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;  in perf_output_copy_aux()
598 		addr = rb->aux_pages[from >> PAGE_SHIFT];  in perf_output_copy_aux()
607 		from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;  in perf_output_copy_aux()
641 static void rb_free_aux_page(struct perf_buffer *rb, int idx)  in rb_free_aux_page()  argument
643 	struct page *page = virt_to_page(rb->aux_pages[idx]);  in rb_free_aux_page()
650 static void __rb_free_aux(struct perf_buffer *rb)  in __rb_free_aux()  argument
662 	if (rb->aux_priv) {  in __rb_free_aux()
663 		rb->free_aux(rb->aux_priv);  in __rb_free_aux()
664 		rb->free_aux = NULL;  in __rb_free_aux()
665 		rb->aux_priv = NULL;  in __rb_free_aux()
668 	if (rb->aux_nr_pages) {  in __rb_free_aux()
669 		for (pg = 0; pg < rb->aux_nr_pages; pg++)  in __rb_free_aux()
670 			rb_free_aux_page(rb, pg);  in __rb_free_aux()
672 		kfree(rb->aux_pages);  in __rb_free_aux()
673 		rb->aux_nr_pages = 0;  in __rb_free_aux()
677 int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event,  in rb_alloc_aux()  argument
720 	rb->aux_pages = kcalloc_node(nr_pages, sizeof(void *), GFP_KERNEL,  in rb_alloc_aux()
722 	if (!rb->aux_pages)  in rb_alloc_aux()
725 	rb->free_aux = event->pmu->free_aux;  in rb_alloc_aux()
726 	for (rb->aux_nr_pages = 0; rb->aux_nr_pages < nr_pages;) {  in rb_alloc_aux()
730 		order = min(max_order, ilog2(nr_pages - rb->aux_nr_pages));  in rb_alloc_aux()
735 		for (last = rb->aux_nr_pages + (1 << page_private(page));  in rb_alloc_aux()
736 		     last > rb->aux_nr_pages; rb->aux_nr_pages++)  in rb_alloc_aux()
737 			rb->aux_pages[rb->aux_nr_pages] = page_address(page++);  in rb_alloc_aux()
748 		struct page *page = virt_to_page(rb->aux_pages[0]);  in rb_alloc_aux()
754 	rb->aux_priv = event->pmu->setup_aux(event, rb->aux_pages, nr_pages,  in rb_alloc_aux()
756 	if (!rb->aux_priv)  in rb_alloc_aux()
767 	refcount_set(&rb->aux_refcount, 1);  in rb_alloc_aux()
769 	rb->aux_overwrite = overwrite;  in rb_alloc_aux()
770 	rb->aux_watermark = watermark;  in rb_alloc_aux()
774 		rb->aux_pgoff = pgoff;  in rb_alloc_aux()
776 		__rb_free_aux(rb);  in rb_alloc_aux()
781 void rb_free_aux(struct perf_buffer *rb)  in rb_free_aux()  argument
783 	if (refcount_dec_and_test(&rb->aux_refcount))  in rb_free_aux()
784 		__rb_free_aux(rb);  in rb_free_aux()
794 __perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)  in __perf_mmap_to_page()  argument
796 	if (pgoff > rb->nr_pages)  in __perf_mmap_to_page()
800 		return virt_to_page(rb->user_page);  in __perf_mmap_to_page()
802 	return virt_to_page(rb->data_pages[pgoff - 1]);  in __perf_mmap_to_page()
828 	struct perf_buffer *rb;  in rb_alloc()  local
839 	rb = kzalloc_node(size, GFP_KERNEL, node);  in rb_alloc()
840 	if (!rb)  in rb_alloc()
843 	rb->user_page = perf_mmap_alloc_page(cpu);  in rb_alloc()
844 	if (!rb->user_page)  in rb_alloc()
848 		rb->data_pages[i] = perf_mmap_alloc_page(cpu);  in rb_alloc()
849 		if (!rb->data_pages[i])  in rb_alloc()
853 	rb->nr_pages = nr_pages;  in rb_alloc()
855 	ring_buffer_init(rb, watermark, flags);  in rb_alloc()
857 	return rb;  in rb_alloc()
861 		perf_mmap_free_page(rb->data_pages[i]);  in rb_alloc()
863 	perf_mmap_free_page(rb->user_page);  in rb_alloc()
866 	kfree(rb);  in rb_alloc()
872 void rb_free(struct perf_buffer *rb)  in rb_free()  argument
876 	perf_mmap_free_page(rb->user_page);  in rb_free()
877 	for (i = 0; i < rb->nr_pages; i++)  in rb_free()
878 		perf_mmap_free_page(rb->data_pages[i]);  in rb_free()
879 	kfree(rb);  in rb_free()
884 __perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)  in __perf_mmap_to_page()  argument
887 	if (pgoff > data_page_nr(rb))  in __perf_mmap_to_page()
890 	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);  in __perf_mmap_to_page()
902 	struct perf_buffer *rb;  in rb_free_work()  local
906 	rb = container_of(work, struct perf_buffer, work);  in rb_free_work()
907 	nr = data_page_nr(rb);  in rb_free_work()
909 	base = rb->user_page;  in rb_free_work()
915 	kfree(rb);  in rb_free_work()
918 void rb_free(struct perf_buffer *rb)  in rb_free()  argument
920 	schedule_work(&rb->work);  in rb_free()
925 	struct perf_buffer *rb;  in rb_alloc()  local
934 	rb = kzalloc_node(size, GFP_KERNEL, node);  in rb_alloc()
935 	if (!rb)  in rb_alloc()
938 	INIT_WORK(&rb->work, rb_free_work);  in rb_alloc()
944 	rb->user_page = all_buf;  in rb_alloc()
945 	rb->data_pages[0] = all_buf + PAGE_SIZE;  in rb_alloc()
947 		rb->nr_pages = 1;  in rb_alloc()
948 		rb->page_order = ilog2(nr_pages);  in rb_alloc()
951 	ring_buffer_init(rb, watermark, flags);  in rb_alloc()
953 	return rb;  in rb_alloc()
956 	kfree(rb);  in rb_alloc()
965 perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)  in perf_mmap_to_page()  argument
967 	if (rb->aux_nr_pages) {  in perf_mmap_to_page()
969 		if (pgoff > rb->aux_pgoff + rb->aux_nr_pages)  in perf_mmap_to_page()
973 		if (pgoff >= rb->aux_pgoff) {  in perf_mmap_to_page()
974 			int aux_pgoff = array_index_nospec(pgoff - rb->aux_pgoff, rb->aux_nr_pages);  in perf_mmap_to_page()
975 			return virt_to_page(rb->aux_pages[aux_pgoff]);  in perf_mmap_to_page()
979 	return __perf_mmap_to_page(rb, pgoff);  in perf_mmap_to_page()