1 // SPDX-License-Identifier: GPL-2.0
6  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
75 	struct task_struct *p = tfc->p;  in remote_function()
78 		/* -EAGAIN */  in remote_function()
87 		tfc->ret = -ESRCH; /* No such (running) process */  in remote_function()
92 	tfc->ret = tfc->func(tfc->info);  in remote_function()
96  * task_function_call - call a function on the cpu on which a task runs
106  * returns @func return value or -ESRCH or -ENXIO when the process isn't running
115 		.ret	= -EAGAIN,  in task_function_call()
125 		if (ret != -EAGAIN)  in task_function_call()
135  * cpu_function_call - call a function on the cpu
142  * returns: @func return value or -ENXIO when the cpu is offline
150 		.ret	= -ENXIO, /* No such CPU */  in cpu_function_call()
174 	raw_spin_lock(&ctx->lock);  in __perf_ctx_lock()
175 	WARN_ON_ONCE(ctx->is_active & EVENT_FROZEN);  in __perf_ctx_lock()
181 	__perf_ctx_lock(&cpuctx->ctx);  in perf_ctx_lock()
192 	if (ctx->is_active & EVENT_FROZEN) {  in __perf_ctx_unlock()
193 		if (!(ctx->is_active & EVENT_ALL))  in __perf_ctx_unlock()
194 			ctx->is_active = 0;  in __perf_ctx_unlock()
196 			ctx->is_active &= ~EVENT_FROZEN;  in __perf_ctx_unlock()
198 	raw_spin_unlock(&ctx->lock);  in __perf_ctx_unlock()
206 	__perf_ctx_unlock(&cpuctx->ctx);  in perf_ctx_unlock()
209 #define TASK_TOMBSTONE ((void *)-1L)
213 	return READ_ONCE(event->owner) == TASK_TOMBSTONE;  in is_kernel_event()
221 	return this_cpu_ptr(&perf_cpu_context)->task_ctx;  in perf_cpu_task_ctx()
227  * When !ctx->nr_events a task context will not be scheduled. This means
233  *  - removing the last event from a task ctx; this is relatively straight
236  *  - adding the first event to a task ctx; this is tricky because we cannot
237  *    rely on ctx->is_active and therefore cannot use event_function_call().
240  * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
255 	struct perf_event *event = efs->event;  in event_function()
256 	struct perf_event_context *ctx = event->ctx;  in event_function()
258 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in event_function()
265 	 * Since we do the IPI call without holding ctx->lock things can have  in event_function()
268 	if (ctx->task) {  in event_function()
269 		if (ctx->task != current) {  in event_function()
270 			ret = -ESRCH;  in event_function()
278 		 * above ctx->task != current test), therefore we must have  in event_function()
279 		 * ctx->is_active here.  in event_function()
281 		WARN_ON_ONCE(!ctx->is_active);  in event_function()
283 		 * And since we have ctx->is_active, cpuctx->task_ctx must  in event_function()
288 		WARN_ON_ONCE(&cpuctx->ctx != ctx);  in event_function()
291 	efs->func(event, cpuctx, ctx, efs->data);  in event_function()
300 	struct perf_event_context *ctx = event->ctx;  in event_function_call()
301 	struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */  in event_function_call()
309 	if (!event->parent) {  in event_function_call()
312 		 * stabilize the event->ctx relation. See  in event_function_call()
315 		lockdep_assert_held(&ctx->mutex);  in event_function_call()
319 		cpu_function_call(event->cpu, event_function, &efs);  in event_function_call()
337 	task = ctx->task;  in event_function_call()
340 	if (ctx->is_active) {  in event_function_call()
357 	struct perf_event_context *ctx = event->ctx;  in event_function_local()
359 	struct task_struct *task = READ_ONCE(ctx->task);  in event_function_local()
373 	task = ctx->task;  in event_function_local()
380 		 * otherwise we're screwed, since we cannot IPI to somewhere  in event_function_local()
383 		if (ctx->is_active) {  in event_function_local()
387 			if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))  in event_function_local()
391 		WARN_ON_ONCE(&cpuctx->ctx != ctx);  in event_function_local()
448  *  -1 - not paranoid at all
449  *   0 - disallow raw tracepoint access for unpriv
450  *   1 - disallow cpu events for unpriv
451  *   2 - disallow kernel profiling for unpriv
496 		return -EINVAL;  in perf_event_max_sample_rate_handler()
566 	running_len -= running_len/NR_ACCUMULATED_SAMPLES;  in perf_sample_event_took()
620 	return event->clock();  in perf_event_clock()
626  * The basic idea is to use event->state to determine which (if any) time
648 	struct perf_event *leader = event->group_leader;  in __perf_effective_state()
650 	if (leader->state <= PERF_EVENT_STATE_OFF)  in __perf_effective_state()
651 		return leader->state;  in __perf_effective_state()
653 	return event->state;  in __perf_effective_state()
660 	u64 delta = now - event->tstamp;  in __perf_update_times()
662 	*enabled = event->total_time_enabled;  in __perf_update_times()
666 	*running = event->total_time_running;  in __perf_update_times()
675 	__perf_update_times(event, now, &event->total_time_enabled,  in perf_event_update_time()
676 					&event->total_time_running);  in perf_event_update_time()
677 	event->tstamp = now;  in perf_event_update_time()
691 	if (event->state == state)  in perf_event_set_state()
699 	if ((event->state < 0) ^ (state < 0))  in perf_event_set_state()
702 	WRITE_ONCE(event->state, state);  in perf_event_set_state()
706  * UP store-release, load-acquire
723 	list_for_each_entry(_epc, &((_ctx)->pmu_ctx_list), pmu_ctx_entry) \
724 		if (_cgroup && !_epc->nr_cgroups)			\
726 		else if (_pmu && _epc->pmu != _pmu)			\
735 		perf_pmu_disable(pmu_ctx->pmu);  in perf_ctx_disable()
743 		perf_pmu_enable(pmu_ctx->pmu);  in perf_ctx_enable()
757 	if (!event->cgrp)  in perf_cgroup_match()
761 	if (!cpuctx->cgrp)  in perf_cgroup_match()
770 	return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,  in perf_cgroup_match()
771 				    event->cgrp->css.cgroup);  in perf_cgroup_match()
776 	css_put(&event->cgrp->css);  in perf_detach_cgroup()
777 	event->cgrp = NULL;  in perf_detach_cgroup()
782 	return event->cgrp != NULL;  in is_cgroup_event()
789 	t = per_cpu_ptr(event->cgrp->info, event->cpu);  in perf_cgroup_event_time()
790 	return t->time;  in perf_cgroup_event_time()
797 	t = per_cpu_ptr(event->cgrp->info, event->cpu);  in perf_cgroup_event_time_now()
798 	if (!__load_acquire(&t->active))  in perf_cgroup_event_time_now()
799 		return t->time;  in perf_cgroup_event_time_now()
800 	now += READ_ONCE(t->timeoffset);  in perf_cgroup_event_time_now()
807 		info->time += now - info->timestamp;  in __update_cgrp_time()
808 	info->timestamp = now;  in __update_cgrp_time()
812 	WRITE_ONCE(info->timeoffset, info->time - info->timestamp);  in __update_cgrp_time()
817 	struct perf_cgroup *cgrp = cpuctx->cgrp;  in update_cgrp_time_from_cpuctx()
824 		for (css = &cgrp->css; css; css = css->parent) {  in update_cgrp_time_from_cpuctx()
826 			info = this_cpu_ptr(cgrp->info);  in update_cgrp_time_from_cpuctx()
830 				__store_release(&info->active, 0);  in update_cgrp_time_from_cpuctx()
846 	info = this_cpu_ptr(event->cgrp->info);  in update_cgrp_time_from_event()
850 	if (info->active)  in update_cgrp_time_from_event()
857 	struct perf_event_context *ctx = &cpuctx->ctx;  in perf_cgroup_set_timestamp()
858 	struct perf_cgroup *cgrp = cpuctx->cgrp;  in perf_cgroup_set_timestamp()
863 	 * ctx->lock held by caller  in perf_cgroup_set_timestamp()
870 	WARN_ON_ONCE(!ctx->nr_cgroups);  in perf_cgroup_set_timestamp()
872 	for (css = &cgrp->css; css; css = css->parent) {  in perf_cgroup_set_timestamp()
874 		info = this_cpu_ptr(cgrp->info);  in perf_cgroup_set_timestamp()
875 		__update_cgrp_time(info, ctx->timestamp, false);  in perf_cgroup_set_timestamp()
876 		__store_release(&info->active, 1);  in perf_cgroup_set_timestamp()
889 	 * cpuctx->cgrp is set when the first cgroup event enabled,  in perf_cgroup_switch()
892 	if (READ_ONCE(cpuctx->cgrp) == NULL)  in perf_cgroup_switch()
895 	WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);  in perf_cgroup_switch()
898 	if (READ_ONCE(cpuctx->cgrp) == cgrp)  in perf_cgroup_switch()
901 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_cgroup_switch()
902 	perf_ctx_disable(&cpuctx->ctx, true);  in perf_cgroup_switch()
904 	ctx_sched_out(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP);  in perf_cgroup_switch()
910 	cpuctx->cgrp = cgrp;  in perf_cgroup_switch()
916 	ctx_sched_in(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP);  in perf_cgroup_switch()
918 	perf_ctx_enable(&cpuctx->ctx, true);  in perf_cgroup_switch()
919 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_cgroup_switch()
933 	for (heap_size = 1; css; css = css->parent)  in perf_cgroup_ensure_storage()
938 		if (heap_size <= cpuctx->heap_size)  in perf_cgroup_ensure_storage()
944 			ret = -ENOMEM;  in perf_cgroup_ensure_storage()
948 		raw_spin_lock_irq(&cpuctx->ctx.lock);  in perf_cgroup_ensure_storage()
949 		if (cpuctx->heap_size < heap_size) {  in perf_cgroup_ensure_storage()
950 			swap(cpuctx->heap, storage);  in perf_cgroup_ensure_storage()
951 			if (storage == cpuctx->heap_default)  in perf_cgroup_ensure_storage()
953 			cpuctx->heap_size = heap_size;  in perf_cgroup_ensure_storage()
955 		raw_spin_unlock_irq(&cpuctx->ctx.lock);  in perf_cgroup_ensure_storage()
973 		return -EBADF;  in perf_cgroup_connect()
975 	css = css_tryget_online_from_dir(fd_file(f)->f_path.dentry,  in perf_cgroup_connect()
987 	event->cgrp = cgrp;  in perf_cgroup_connect()
994 	if (group_leader && group_leader->cgrp != cgrp) {  in perf_cgroup_connect()
996 		ret = -EINVAL;  in perf_cgroup_connect()
1011 	event->pmu_ctx->nr_cgroups++;  in perf_cgroup_event_enable()
1014 	 * Because cgroup events are always per-cpu events,  in perf_cgroup_event_enable()
1015 	 * @ctx == &cpuctx->ctx.  in perf_cgroup_event_enable()
1019 	if (ctx->nr_cgroups++)  in perf_cgroup_event_enable()
1022 	cpuctx->cgrp = perf_cgroup_from_task(current, ctx);  in perf_cgroup_event_enable()
1033 	event->pmu_ctx->nr_cgroups--;  in perf_cgroup_event_disable()
1036 	 * Because cgroup events are always per-cpu events,  in perf_cgroup_event_disable()
1037 	 * @ctx == &cpuctx->ctx.  in perf_cgroup_event_disable()
1041 	if (--ctx->nr_cgroups)  in perf_cgroup_event_disable()
1044 	cpuctx->cgrp = NULL;  in perf_cgroup_event_disable()
1076 	return -EINVAL;  in perf_cgroup_connect()
1127 	raw_spin_lock(&cpc->hrtimer_lock);  in perf_mux_hrtimer_handler()
1129 		hrtimer_forward_now(hr, cpc->hrtimer_interval);  in perf_mux_hrtimer_handler()
1131 		cpc->hrtimer_active = 0;  in perf_mux_hrtimer_handler()
1132 	raw_spin_unlock(&cpc->hrtimer_lock);  in perf_mux_hrtimer_handler()
1139 	struct hrtimer *timer = &cpc->hrtimer;  in __perf_mux_hrtimer_init()
1140 	struct pmu *pmu = cpc->epc.pmu;  in __perf_mux_hrtimer_init()
1147 	interval = pmu->hrtimer_interval_ms;  in __perf_mux_hrtimer_init()
1149 		interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;  in __perf_mux_hrtimer_init()
1151 	cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);  in __perf_mux_hrtimer_init()
1153 	raw_spin_lock_init(&cpc->hrtimer_lock);  in __perf_mux_hrtimer_init()
1155 	timer->function = perf_mux_hrtimer_handler;  in __perf_mux_hrtimer_init()
1160 	struct hrtimer *timer = &cpc->hrtimer;  in perf_mux_hrtimer_restart()
1163 	raw_spin_lock_irqsave(&cpc->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
1164 	if (!cpc->hrtimer_active) {  in perf_mux_hrtimer_restart()
1165 		cpc->hrtimer_active = 1;  in perf_mux_hrtimer_restart()
1166 		hrtimer_forward_now(timer, cpc->hrtimer_interval);  in perf_mux_hrtimer_restart()
1169 	raw_spin_unlock_irqrestore(&cpc->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
1181 	int *count = this_cpu_ptr(pmu->pmu_disable_count);  in perf_pmu_disable()
1183 		pmu->pmu_disable(pmu);  in perf_pmu_disable()
1188 	int *count = this_cpu_ptr(pmu->pmu_disable_count);  in perf_pmu_enable()
1189 	if (!--(*count))  in perf_pmu_enable()
1190 		pmu->pmu_enable(pmu);  in perf_pmu_enable()
1195 	WARN_ON_ONCE(*this_cpu_ptr(pmu->pmu_disable_count) == 0);  in perf_assert_pmu_disabled()
1200 	refcount_inc(&ctx->refcount);  in get_ctx()
1205 	if (pmu->task_ctx_cache)  in alloc_task_ctx_data()
1206 		return kmem_cache_zalloc(pmu->task_ctx_cache, GFP_KERNEL);  in alloc_task_ctx_data()
1213 	if (pmu->task_ctx_cache && task_ctx_data)  in free_task_ctx_data()
1214 		kmem_cache_free(pmu->task_ctx_cache, task_ctx_data);  in free_task_ctx_data()
1227 	if (refcount_dec_and_test(&ctx->refcount)) {  in put_ctx()
1228 		if (ctx->parent_ctx)  in put_ctx()
1229 			put_ctx(ctx->parent_ctx);  in put_ctx()
1230 		if (ctx->task && ctx->task != TASK_TOMBSTONE)  in put_ctx()
1231 			put_task_struct(ctx->task);  in put_ctx()
1232 		call_rcu(&ctx->rcu_head, free_ctx);  in put_ctx()
1246  *  - perf_event_exit_task_context()	[ child , 0 ]
1250  *  - perf_event_init_context()		[ parent, 0 ]
1258  * While it appears there is an obvious deadlock here -- the parent and child
1260  * life-time rules separate them. That is an exiting task cannot fork, and a
1263  * But remember that these are parent<->child context relations, and
1269  * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
1280  * means that only external vectors (perf_fops, prctl) can perturb the event
1284  * However; because event->ctx can change while we're waiting to acquire
1285  * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
1301  *	  cpuctx->mutex / perf_event_context::mutex
1310 	ctx = READ_ONCE(event->ctx);  in perf_event_ctx_lock_nested()
1311 	if (!refcount_inc_not_zero(&ctx->refcount)) {  in perf_event_ctx_lock_nested()
1317 	mutex_lock_nested(&ctx->mutex, nesting);  in perf_event_ctx_lock_nested()
1318 	if (event->ctx != ctx) {  in perf_event_ctx_lock_nested()
1319 		mutex_unlock(&ctx->mutex);  in perf_event_ctx_lock_nested()
1336 	mutex_unlock(&ctx->mutex);  in perf_event_ctx_unlock()
1341  * This must be done under the ctx->lock, such as to serialize against
1343  * calling scheduler related locks and ctx->lock nests inside those.
1348 	struct perf_event_context *parent_ctx = ctx->parent_ctx;  in unclone_ctx()
1350 	lockdep_assert_held(&ctx->lock);  in unclone_ctx()
1353 		ctx->parent_ctx = NULL;  in unclone_ctx()
1354 	ctx->generation++;  in unclone_ctx()
1366 	if (event->parent)  in perf_event_pid_type()
1367 		event = event->parent;  in perf_event_pid_type()
1369 	nr = __task_pid_nr_ns(p, type, event->ns);  in perf_event_pid_type()
1370 	/* avoid -1 if it is idle thread or runs in another ns */  in perf_event_pid_type()
1372 		nr = -1;  in perf_event_pid_type()
1392 	u64 id = event->id;  in primary_event_id()
1394 	if (event->parent)  in primary_event_id()
1395 		id = event->parent->id;  in primary_event_id()
1415 	 * part of the read side critical section was irqs-enabled -- see  in perf_lock_task_context()
1418 	 * Since ctx->lock nests under rq->lock we must ensure the entire read  in perf_lock_task_context()
1423 	ctx = rcu_dereference(task->perf_event_ctxp);  in perf_lock_task_context()
1435 		raw_spin_lock(&ctx->lock);  in perf_lock_task_context()
1436 		if (ctx != rcu_dereference(task->perf_event_ctxp)) {  in perf_lock_task_context()
1437 			raw_spin_unlock(&ctx->lock);  in perf_lock_task_context()
1443 		if (ctx->task == TASK_TOMBSTONE ||  in perf_lock_task_context()
1444 		    !refcount_inc_not_zero(&ctx->refcount)) {  in perf_lock_task_context()
1445 			raw_spin_unlock(&ctx->lock);  in perf_lock_task_context()
1448 			WARN_ON_ONCE(ctx->task != task);  in perf_lock_task_context()
1470 		++ctx->pin_count;  in perf_pin_task_context()
1471 		raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_pin_task_context()
1480 	raw_spin_lock_irqsave(&ctx->lock, flags);  in perf_unpin_context()
1481 	--ctx->pin_count;  in perf_unpin_context()
1482 	raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_unpin_context()
1492 	lockdep_assert_held(&ctx->lock);  in __update_context_time()
1495 		ctx->time += now - ctx->timestamp;  in __update_context_time()
1496 	ctx->timestamp = now;  in __update_context_time()
1499 	 * The above: time' = time + (now - timestamp), can be re-arranged  in __update_context_time()
1500 	 * into: time` = now + (time - timestamp), which gives a single value  in __update_context_time()
1504 	 * it's (obviously) not possible to acquire ctx->lock in order to read  in __update_context_time()
1507 	WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp);  in __update_context_time()
1517 	struct perf_event_context *ctx = event->ctx;  in perf_event_time()
1525 	return ctx->time;  in perf_event_time()
1530 	struct perf_event_context *ctx = event->ctx;  in perf_event_time_now()
1538 	if (!(__load_acquire(&ctx->is_active) & EVENT_TIME))  in perf_event_time_now()
1539 		return ctx->time;  in perf_event_time_now()
1541 	now += READ_ONCE(ctx->timeoffset);  in perf_event_time_now()
1547 	struct perf_event_context *ctx = event->ctx;  in get_event_type()
1550 	lockdep_assert_held(&ctx->lock);  in get_event_type()
1556 	if (event->group_leader != event)  in get_event_type()
1557 		event = event->group_leader;  in get_event_type()
1559 	event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE;  in get_event_type()
1560 	if (!ctx->task)  in get_event_type()
1571 	RB_CLEAR_NODE(&event->group_node);  in init_event_group()
1572 	event->group_index = 0;  in init_event_group()
1582 	if (event->attr.pinned)  in get_event_groups()
1583 		return &ctx->pinned_groups;  in get_event_groups()
1585 		return &ctx->flexible_groups;  in get_event_groups()
1593 	groups->tree = RB_ROOT;  in perf_event_groups_init()
1594 	groups->index = 0;  in perf_event_groups_init()
1602 	if (event->cgrp)  in event_cgroup()
1603 		cgroup = event->cgrp->css.cgroup;  in event_cgroup()
1620 	if (left_cpu < right->cpu)  in perf_event_groups_cmp()
1621 		return -1;  in perf_event_groups_cmp()
1622 	if (left_cpu > right->cpu)  in perf_event_groups_cmp()
1626 		if (left_pmu < right->pmu_ctx->pmu)  in perf_event_groups_cmp()
1627 			return -1;  in perf_event_groups_cmp()
1628 		if (left_pmu > right->pmu_ctx->pmu)  in perf_event_groups_cmp()
1642 				return -1;  in perf_event_groups_cmp()
1653 				return -1;  in perf_event_groups_cmp()
1660 	if (left_group_index < right->group_index)  in perf_event_groups_cmp()
1661 		return -1;  in perf_event_groups_cmp()
1662 	if (left_group_index > right->group_index)  in perf_event_groups_cmp()
1674 	return perf_event_groups_cmp(e->cpu, e->pmu_ctx->pmu, event_cgroup(e),  in __group_less()
1675 				     e->group_index, __node_2_pe(b)) < 0;  in __group_less()
1690 	return perf_event_groups_cmp(a->cpu, a->pmu, a->cgroup, b->group_index, b);  in __group_cmp()
1700 	return perf_event_groups_cmp(a->cpu, a->pmu, event_cgroup(b),  in __group_cmp_ignore_cgroup()
1701 				     b->group_index, b);  in __group_cmp_ignore_cgroup()
1706  *   {@event->cpu, @event->pmu_ctx->pmu, event_cgroup(@event), ++@groups->index}
1713 	event->group_index = ++groups->index;  in perf_event_groups_insert()
1715 	rb_add(&event->group_node, &groups->tree, __group_less);  in perf_event_groups_insert()
1737 	WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) ||  in perf_event_groups_delete()
1738 		     RB_EMPTY_ROOT(&groups->tree));  in perf_event_groups_delete()
1740 	rb_erase(&event->group_node, &groups->tree);  in perf_event_groups_delete()
1770 	node = rb_find_first(&key, &groups->tree, __group_cmp);  in perf_event_groups_first()
1781 		.cpu = event->cpu,  in perf_event_groups_next()
1787 	next = rb_next_match(&key, &event->group_node, __group_cmp);  in perf_event_groups_next()
1802 	for (event = rb_entry_safe(rb_first(&((groups)->tree)),		\
1804 		event = rb_entry_safe(rb_next(&event->group_node),	\
1812 	return attr->inherit && (attr->sample_type & PERF_SAMPLE_READ);  in has_inherit_and_sample_read()
1817  * Must be called with ctx->mutex and ctx->lock held.
1822 	lockdep_assert_held(&ctx->lock);  in list_add_event()
1824 	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);  in list_add_event()
1825 	event->attach_state |= PERF_ATTACH_CONTEXT;  in list_add_event()
1827 	event->tstamp = perf_event_time(event);  in list_add_event()
1834 	if (event->group_leader == event) {  in list_add_event()
1835 		event->group_caps = event->event_caps;  in list_add_event()
1839 	list_add_rcu(&event->event_entry, &ctx->event_list);  in list_add_event()
1840 	ctx->nr_events++;  in list_add_event()
1841 	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)  in list_add_event()
1842 		ctx->nr_user++;  in list_add_event()
1843 	if (event->attr.inherit_stat)  in list_add_event()
1844 		ctx->nr_stat++;  in list_add_event()
1845 	if (has_inherit_and_sample_read(&event->attr))  in list_add_event()
1846 		local_inc(&ctx->nr_no_switch_fast);  in list_add_event()
1848 	if (event->state > PERF_EVENT_STATE_OFF)  in list_add_event()
1851 	ctx->generation++;  in list_add_event()
1852 	event->pmu_ctx->nr_events++;  in list_add_event()
1860 	event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :  in perf_event__state_init()
1900 		size += sizeof(data->ip);  in __perf_event_header_size()
1903 		size += sizeof(data->addr);  in __perf_event_header_size()
1906 		size += sizeof(data->period);  in __perf_event_header_size()
1909 		size += sizeof(data->weight.full);  in __perf_event_header_size()
1912 		size += event->read_size;  in __perf_event_header_size()
1915 		size += sizeof(data->data_src.val);  in __perf_event_header_size()
1918 		size += sizeof(data->txn);  in __perf_event_header_size()
1921 		size += sizeof(data->phys_addr);  in __perf_event_header_size()
1924 		size += sizeof(data->cgroup);  in __perf_event_header_size()
1927 		size += sizeof(data->data_page_size);  in __perf_event_header_size()
1930 		size += sizeof(data->code_page_size);  in __perf_event_header_size()
1932 	event->header_size = size;  in __perf_event_header_size()
1941 	event->read_size =  in perf_event__header_size()
1942 		__perf_event_read_size(event->attr.read_format,  in perf_event__header_size()
1943 				       event->group_leader->nr_siblings);  in perf_event__header_size()
1944 	__perf_event_header_size(event, event->attr.sample_type);  in perf_event__header_size()
1950 	u64 sample_type = event->attr.sample_type;  in perf_event__id_header_size()
1954 		size += sizeof(data->tid_entry);  in perf_event__id_header_size()
1957 		size += sizeof(data->time);  in perf_event__id_header_size()
1960 		size += sizeof(data->id);  in perf_event__id_header_size()
1963 		size += sizeof(data->id);  in perf_event__id_header_size()
1966 		size += sizeof(data->stream_id);  in perf_event__id_header_size()
1969 		size += sizeof(data->cpu_entry);  in perf_event__id_header_size()
1971 	event->id_header_size = size;  in perf_event__id_header_size()
1980  * depends on per-event read_format, also (re)check the existing events.
1987 	struct perf_event *sibling, *group_leader = event->group_leader;  in perf_event_validate_size()
1989 	if (__perf_event_read_size(event->attr.read_format,  in perf_event_validate_size()
1990 				   group_leader->nr_siblings + 1) > 16*1024)  in perf_event_validate_size()
1993 	if (__perf_event_read_size(group_leader->attr.read_format,  in perf_event_validate_size()
1994 				   group_leader->nr_siblings + 1) > 16*1024)  in perf_event_validate_size()
1998 	 * When creating a new group leader, group_leader->ctx is initialized  in perf_event_validate_size()
2000 	 * for_each_sibling_event() until group_leader->ctx is set. A new group  in perf_event_validate_size()
2002 	 * the non-existent siblings.  in perf_event_validate_size()
2008 		if (__perf_event_read_size(sibling->attr.read_format,  in perf_event_validate_size()
2009 					   group_leader->nr_siblings + 1) > 16*1024)  in perf_event_validate_size()
2018 	struct perf_event *group_leader = event->group_leader, *pos;  in perf_group_attach()
2020 	lockdep_assert_held(&event->ctx->lock);  in perf_group_attach()
2026 	if (event->attach_state & PERF_ATTACH_GROUP)  in perf_group_attach()
2029 	event->attach_state |= PERF_ATTACH_GROUP;  in perf_group_attach()
2034 	WARN_ON_ONCE(group_leader->ctx != event->ctx);  in perf_group_attach()
2036 	group_leader->group_caps &= event->event_caps;  in perf_group_attach()
2038 	list_add_tail(&event->sibling_list, &group_leader->sibling_list);  in perf_group_attach()
2039 	group_leader->nr_siblings++;  in perf_group_attach()
2040 	group_leader->group_generation++;  in perf_group_attach()
2050  * Must be called with ctx->mutex and ctx->lock held.
2055 	WARN_ON_ONCE(event->ctx != ctx);  in list_del_event()
2056 	lockdep_assert_held(&ctx->lock);  in list_del_event()
2059 	 * We can have double detach due to exit/hot-unplug + close.  in list_del_event()
2061 	if (!(event->attach_state & PERF_ATTACH_CONTEXT))  in list_del_event()
2064 	event->attach_state &= ~PERF_ATTACH_CONTEXT;  in list_del_event()
2066 	ctx->nr_events--;  in list_del_event()
2067 	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)  in list_del_event()
2068 		ctx->nr_user--;  in list_del_event()
2069 	if (event->attr.inherit_stat)  in list_del_event()
2070 		ctx->nr_stat--;  in list_del_event()
2071 	if (has_inherit_and_sample_read(&event->attr))  in list_del_event()
2072 		local_dec(&ctx->nr_no_switch_fast);  in list_del_event()
2074 	list_del_rcu(&event->event_entry);  in list_del_event()
2076 	if (event->group_leader == event)  in list_del_event()
2083 	 * of error state is by explicit re-enabling  in list_del_event()
2086 	if (event->state > PERF_EVENT_STATE_OFF) {  in list_del_event()
2091 	ctx->generation++;  in list_del_event()
2092 	event->pmu_ctx->nr_events--;  in list_del_event()
2101 	if (!event->pmu->aux_output_match)  in perf_aux_output_match()
2104 	return event->pmu->aux_output_match(aux_event);  in perf_aux_output_match()
2113 	struct perf_event_context *ctx = event->ctx;  in perf_put_aux_event()
2119 	if (event->aux_event) {  in perf_put_aux_event()
2120 		iter = event->aux_event;  in perf_put_aux_event()
2121 		event->aux_event = NULL;  in perf_put_aux_event()
2130 	for_each_sibling_event(iter, event->group_leader) {  in perf_put_aux_event()
2131 		if (iter->aux_event != event)  in perf_put_aux_event()
2134 		iter->aux_event = NULL;  in perf_put_aux_event()
2149 	return !!event->attr.aux_output || !!event->attr.aux_sample_size;  in perf_need_aux_event()
2167 	if (event->attr.aux_output && event->attr.aux_sample_size)  in perf_get_aux_event()
2170 	if (event->attr.aux_output &&  in perf_get_aux_event()
2174 	if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux)  in perf_get_aux_event()
2177 	if (!atomic_long_inc_not_zero(&group_leader->refcount))  in perf_get_aux_event()
2186 	event->aux_event = group_leader;  in perf_get_aux_event()
2193 	return event->attr.pinned ? &event->pmu_ctx->pinned_active :  in get_event_list()
2194 				    &event->pmu_ctx->flexible_active;  in get_event_list()
2205 	event_sched_out(event, event->ctx);  in perf_remove_sibling_event()
2211 	struct perf_event *leader = event->group_leader;  in perf_group_detach()
2213 	struct perf_event_context *ctx = event->ctx;  in perf_group_detach()
2215 	lockdep_assert_held(&ctx->lock);  in perf_group_detach()
2218 	 * We can have double detach due to exit/hot-unplug + close.  in perf_group_detach()
2220 	if (!(event->attach_state & PERF_ATTACH_GROUP))  in perf_group_detach()
2223 	event->attach_state &= ~PERF_ATTACH_GROUP;  in perf_group_detach()
2231 		list_del_init(&event->sibling_list);  in perf_group_detach()
2232 		event->group_leader->nr_siblings--;  in perf_group_detach()
2233 		event->group_leader->group_generation++;  in perf_group_detach()
2242 	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {  in perf_group_detach()
2244 		if (sibling->event_caps & PERF_EV_CAP_SIBLING)  in perf_group_detach()
2247 		sibling->group_leader = sibling;  in perf_group_detach()
2248 		list_del_init(&sibling->sibling_list);  in perf_group_detach()
2251 		sibling->group_caps = event->group_caps;  in perf_group_detach()
2253 		if (sibling->attach_state & PERF_ATTACH_CONTEXT) {  in perf_group_detach()
2254 			add_event_to_groups(sibling, event->ctx);  in perf_group_detach()
2256 			if (sibling->state == PERF_EVENT_STATE_ACTIVE)  in perf_group_detach()
2257 				list_add_tail(&sibling->active_list, get_event_list(sibling));  in perf_group_detach()
2260 		WARN_ON_ONCE(sibling->ctx != event->ctx);  in perf_group_detach()
2274 	struct perf_event *parent_event = event->parent;  in perf_child_detach()
2276 	if (!(event->attach_state & PERF_ATTACH_CHILD))  in perf_child_detach()
2279 	event->attach_state &= ~PERF_ATTACH_CHILD;  in perf_child_detach()
2284 	lockdep_assert_held(&parent_event->child_mutex);  in perf_child_detach()
2287 	list_del_init(&event->child_list);  in perf_child_detach()
2292 	return event->state == PERF_EVENT_STATE_DEAD;  in is_orphaned_event()
2298 	return (event->cpu == -1 || event->cpu == smp_processor_id()) &&  in event_filter_match()
2305 	struct perf_event_pmu_context *epc = event->pmu_ctx;  in event_sched_out()
2306 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);  in event_sched_out()
2309 	// XXX cpc serialization, probably per-cpu IRQ disabled  in event_sched_out()
2311 	WARN_ON_ONCE(event->ctx != ctx);  in event_sched_out()
2312 	lockdep_assert_held(&ctx->lock);  in event_sched_out()
2314 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in event_sched_out()
2322 	list_del_init(&event->active_list);  in event_sched_out()
2324 	perf_pmu_disable(event->pmu);  in event_sched_out()
2326 	event->pmu->del(event, 0);  in event_sched_out()
2327 	event->oncpu = -1;  in event_sched_out()
2329 	if (event->pending_disable) {  in event_sched_out()
2330 		event->pending_disable = 0;  in event_sched_out()
2338 		cpc->active_oncpu--;  in event_sched_out()
2339 	if (event->attr.freq && event->attr.sample_freq) {  in event_sched_out()
2340 		ctx->nr_freq--;  in event_sched_out()
2341 		epc->nr_freq--;  in event_sched_out()
2343 	if (event->attr.exclusive || !cpc->active_oncpu)  in event_sched_out()
2344 		cpc->exclusive = 0;  in event_sched_out()
2346 	perf_pmu_enable(event->pmu);  in event_sched_out()
2354 	if (group_event->state != PERF_EVENT_STATE_ACTIVE)  in group_sched_out()
2357 	perf_assert_pmu_disabled(group_event->pmu_ctx->pmu);  in group_sched_out()
2371 	if (ctx->is_active & EVENT_TIME) {  in __ctx_time_update()
2372 		if (ctx->is_active & EVENT_FROZEN)  in __ctx_time_update()
2392 	if (ctx->is_active & EVENT_TIME)  in ctx_time_freeze()
2393 		ctx->is_active |= EVENT_FROZEN;  in ctx_time_freeze()
2399 	if (ctx->is_active & EVENT_TIME) {  in ctx_time_update_event()
2400 		if (ctx->is_active & EVENT_FROZEN)  in ctx_time_update_event()
2423 	struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx;  in __perf_remove_from_context()
2433 		event->pending_disable = 1;  in __perf_remove_from_context()
2441 		event->state = PERF_EVENT_STATE_DEAD;  in __perf_remove_from_context()
2443 	if (!pmu_ctx->nr_events) {  in __perf_remove_from_context()
2444 		pmu_ctx->rotate_necessary = 0;  in __perf_remove_from_context()
2446 		if (ctx->task && ctx->is_active) {  in __perf_remove_from_context()
2449 			cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);  in __perf_remove_from_context()
2450 			WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);  in __perf_remove_from_context()
2451 			cpc->task_epc = NULL;  in __perf_remove_from_context()
2455 	if (!ctx->nr_events && ctx->is_active) {  in __perf_remove_from_context()
2456 		if (ctx == &cpuctx->ctx)  in __perf_remove_from_context()
2459 		ctx->is_active = 0;  in __perf_remove_from_context()
2460 		if (ctx->task) {  in __perf_remove_from_context()
2461 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in __perf_remove_from_context()
2462 			cpuctx->task_ctx = NULL;  in __perf_remove_from_context()
2470  * If event->ctx is a cloned context, callers must make sure that
2471  * every task struct that event->ctx->task could possibly point to
2473  * that only calls us on the top-level context, which can't be a clone.
2479 	struct perf_event_context *ctx = event->ctx;  in perf_remove_from_context()
2481 	lockdep_assert_held(&ctx->mutex);  in perf_remove_from_context()
2488 	raw_spin_lock_irq(&ctx->lock);  in perf_remove_from_context()
2489 	if (!ctx->is_active) {  in perf_remove_from_context()
2492 		raw_spin_unlock_irq(&ctx->lock);  in perf_remove_from_context()
2495 	raw_spin_unlock_irq(&ctx->lock);  in perf_remove_from_context()
2508 	if (event->state < PERF_EVENT_STATE_INACTIVE)  in __perf_event_disable()
2511 	perf_pmu_disable(event->pmu_ctx->pmu);  in __perf_event_disable()
2514 	if (event == event->group_leader)  in __perf_event_disable()
2522 	perf_pmu_enable(event->pmu_ctx->pmu);  in __perf_event_disable()
2528  * If event->ctx is a cloned context, callers must make sure that
2529  * every task struct that event->ctx->task could possibly point to
2532  * hold the top-level event's child_mutex, so any descendant that
2535  * When called from perf_pending_disable it's OK because event->ctx
2541 	struct perf_event_context *ctx = event->ctx;  in _perf_event_disable()
2543 	raw_spin_lock_irq(&ctx->lock);  in _perf_event_disable()
2544 	if (event->state <= PERF_EVENT_STATE_OFF) {  in _perf_event_disable()
2545 		raw_spin_unlock_irq(&ctx->lock);  in _perf_event_disable()
2548 	raw_spin_unlock_irq(&ctx->lock);  in _perf_event_disable()
2574 	event->pending_disable = 1;  in perf_event_disable_inatomic()
2575 	irq_work_queue(&event->pending_disable_irq);  in perf_event_disable_inatomic()
2586 	struct perf_event_pmu_context *epc = event->pmu_ctx;  in event_sched_in()
2587 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);  in event_sched_in()
2590 	WARN_ON_ONCE(event->ctx != ctx);  in event_sched_in()
2592 	lockdep_assert_held(&ctx->lock);  in event_sched_in()
2594 	if (event->state <= PERF_EVENT_STATE_OFF)  in event_sched_in()
2597 	WRITE_ONCE(event->oncpu, smp_processor_id());  in event_sched_in()
2601 	 * ->oncpu if it sees ACTIVE.  in event_sched_in()
2611 	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {  in event_sched_in()
2613 		event->hw.interrupts = 0;  in event_sched_in()
2616 	perf_pmu_disable(event->pmu);  in event_sched_in()
2620 	if (event->pmu->add(event, PERF_EF_START)) {  in event_sched_in()
2622 		event->oncpu = -1;  in event_sched_in()
2623 		ret = -EAGAIN;  in event_sched_in()
2628 		cpc->active_oncpu++;  in event_sched_in()
2629 	if (event->attr.freq && event->attr.sample_freq) {  in event_sched_in()
2630 		ctx->nr_freq++;  in event_sched_in()
2631 		epc->nr_freq++;  in event_sched_in()
2633 	if (event->attr.exclusive)  in event_sched_in()
2634 		cpc->exclusive = 1;  in event_sched_in()
2637 	perf_pmu_enable(event->pmu);  in event_sched_in()
2646 	struct pmu *pmu = group_event->pmu_ctx->pmu;  in group_sched_in()
2648 	if (group_event->state == PERF_EVENT_STATE_OFF)  in group_sched_in()
2651 	pmu->start_txn(pmu, PERF_PMU_TXN_ADD);  in group_sched_in()
2666 	if (!pmu->commit_txn(pmu))  in group_sched_in()
2684 	pmu->cancel_txn(pmu);  in group_sched_in()
2685 	return -EAGAIN;  in group_sched_in()
2693 	struct perf_event_pmu_context *epc = event->pmu_ctx;  in group_can_go_on()
2694 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);  in group_can_go_on()
2699 	if (event->group_caps & PERF_EV_CAP_SOFTWARE)  in group_can_go_on()
2705 	if (cpc->exclusive)  in group_can_go_on()
2711 	if (event->attr.exclusive && !list_empty(get_event_list(event)))  in group_can_go_on()
2733 	if (!cpuctx->task_ctx)  in task_ctx_sched_out()
2736 	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))  in task_ctx_sched_out()
2746 	ctx_sched_in(&cpuctx->ctx, pmu, EVENT_PINNED);  in perf_event_sched_in()
2749 	ctx_sched_in(&cpuctx->ctx, pmu, EVENT_FLEXIBLE);  in perf_event_sched_in()
2756  *  - CPU pinned (EVENT_CPU | EVENT_PINNED)
2757  *  - task pinned (EVENT_PINNED)
2758  *  - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE)
2759  *  - task flexible (EVENT_FLEXIBLE).
2785 	for_each_epc(epc, &cpuctx->ctx, pmu, false)  in ctx_resched()
2786 		perf_pmu_disable(epc->pmu);  in ctx_resched()
2790 			perf_pmu_disable(epc->pmu);  in ctx_resched()
2798 	 *  - EVENT_CPU: schedule out corresponding groups;  in ctx_resched()
2799 	 *  - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups;  in ctx_resched()
2800 	 *  - otherwise, do nothing more.  in ctx_resched()
2803 		ctx_sched_out(&cpuctx->ctx, pmu, event_type);  in ctx_resched()
2805 		ctx_sched_out(&cpuctx->ctx, pmu, EVENT_FLEXIBLE);  in ctx_resched()
2809 	for_each_epc(epc, &cpuctx->ctx, pmu, false)  in ctx_resched()
2810 		perf_pmu_enable(epc->pmu);  in ctx_resched()
2814 			perf_pmu_enable(epc->pmu);  in ctx_resched()
2821 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in perf_pmu_resched()
2832  * things like ctx->is_active and cpuctx->task_ctx are set.
2837 	struct perf_event_context *ctx = event->ctx;  in __perf_install_in_context()
2839 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in __perf_install_in_context()
2843 	raw_spin_lock(&cpuctx->ctx.lock);  in __perf_install_in_context()
2844 	if (ctx->task) {  in __perf_install_in_context()
2845 		raw_spin_lock(&ctx->lock);  in __perf_install_in_context()
2848 		reprogram = (ctx->task == current);  in __perf_install_in_context()
2854 		 * If its not running, we don't care, ctx->lock will  in __perf_install_in_context()
2857 		if (task_curr(ctx->task) && !reprogram) {  in __perf_install_in_context()
2858 			ret = -ESRCH;  in __perf_install_in_context()
2862 		WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx);  in __perf_install_in_context()
2864 		raw_spin_lock(&task_ctx->lock);  in __perf_install_in_context()
2868 	if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {  in __perf_install_in_context()
2874 		reprogram = cgroup_is_descendant(cgrp->css.cgroup,  in __perf_install_in_context()
2875 					event->cgrp->css.cgroup);  in __perf_install_in_context()
2882 		ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu,  in __perf_install_in_context()
2907 	struct task_struct *task = READ_ONCE(ctx->task);  in perf_install_in_context()
2909 	lockdep_assert_held(&ctx->mutex);  in perf_install_in_context()
2913 	if (event->cpu != -1)  in perf_install_in_context()
2914 		WARN_ON_ONCE(event->cpu != cpu);  in perf_install_in_context()
2917 	 * Ensures that if we can observe event->ctx, both the event and ctx  in perf_install_in_context()
2920 	smp_store_release(&event->ctx, ctx);  in perf_install_in_context()
2924 	 * without IPI. Except when this is the first event for the context, in  in perf_install_in_context()
2925 	 * that case we need the magic of the IPI to set ctx->is_active.  in perf_install_in_context()
2928 	 * event will issue the IPI and reprogram the hardware.  in perf_install_in_context()
2931 	    ctx->nr_events && !is_cgroup_event(event)) {  in perf_install_in_context()
2932 		raw_spin_lock_irq(&ctx->lock);  in perf_install_in_context()
2933 		if (ctx->task == TASK_TOMBSTONE) {  in perf_install_in_context()
2934 			raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
2938 		raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
2954 	 * Installing events is tricky because we cannot rely on ctx->is_active  in perf_install_in_context()
2955 	 * to be set in case this is the nr_events 0 -> 1 transition.  in perf_install_in_context()
2969 	 * our task->perf_event_ctxp[] store, such that it will in fact take  in perf_install_in_context()
2972 	 * We do this by task_function_call(), if the IPI fails to hit the task  in perf_install_in_context()
2978 	 * This smp_mb() orders the task->perf_event_ctxp[] store with the  in perf_install_in_context()
2979 	 * task_cpu() load, such that if the IPI then does not find the task  in perf_install_in_context()
2988 	raw_spin_lock_irq(&ctx->lock);  in perf_install_in_context()
2989 	task = ctx->task;  in perf_install_in_context()
2993 		 * cannot happen), and we hold ctx->mutex, which serializes us  in perf_install_in_context()
2996 		raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
3000 	 * If the task is not running, ctx->lock will avoid it becoming so,  in perf_install_in_context()
3004 		raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
3008 	raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
3019 	struct perf_event *leader = event->group_leader;  in __perf_event_enable()
3022 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||  in __perf_event_enable()
3023 	    event->state <= PERF_EVENT_STATE_ERROR)  in __perf_event_enable()
3031 	if (!ctx->is_active)  in __perf_event_enable()
3041 	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)  in __perf_event_enable()
3044 	task_ctx = cpuctx->task_ctx;  in __perf_event_enable()
3045 	if (ctx->task)  in __perf_event_enable()
3048 	ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu, get_event_type(event));  in __perf_event_enable()
3054  * If event->ctx is a cloned context, callers must make sure that
3055  * every task struct that event->ctx->task could possibly point to
3062 	struct perf_event_context *ctx = event->ctx;  in _perf_event_enable()
3064 	raw_spin_lock_irq(&ctx->lock);  in _perf_event_enable()
3065 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||  in _perf_event_enable()
3066 	    event->state <  PERF_EVENT_STATE_ERROR) {  in _perf_event_enable()
3068 		raw_spin_unlock_irq(&ctx->lock);  in _perf_event_enable()
3077 	 * been scheduled away before the cross-call arrived.  in _perf_event_enable()
3079 	if (event->state == PERF_EVENT_STATE_ERROR) {  in _perf_event_enable()
3083 		if (event->event_caps & PERF_EV_CAP_SIBLING &&  in _perf_event_enable()
3084 		    event->group_leader == event)  in _perf_event_enable()
3087 		event->state = PERF_EVENT_STATE_OFF;  in _perf_event_enable()
3089 	raw_spin_unlock_irq(&ctx->lock);  in _perf_event_enable()
3115 	struct perf_event *event = sd->event;  in __perf_event_stop()
3118 	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)  in __perf_event_stop()
3128 	if (READ_ONCE(event->oncpu) != smp_processor_id())  in __perf_event_stop()
3129 		return -EAGAIN;  in __perf_event_stop()
3131 	event->pmu->stop(event, PERF_EF_UPDATE);  in __perf_event_stop()
3139 	 * Since this is happening on an event-local CPU, no trace is lost  in __perf_event_stop()
3142 	if (sd->restart)  in __perf_event_stop()
3143 		event->pmu->start(event, 0);  in __perf_event_stop()
3157 		if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)  in perf_event_stop()
3165 		 * inactive here (event->oncpu==-1), there's nothing more to do;  in perf_event_stop()
3166 		 * fall through with ret==-ENXIO.  in perf_event_stop()
3168 		ret = cpu_function_call(READ_ONCE(event->oncpu),  in perf_event_stop()
3170 	} while (ret == -EAGAIN);  in perf_event_stop()
3189  *     pre-existing mappings, called once when new filters arrive via SET_FILTER
3204 	raw_spin_lock(&ifh->lock);  in perf_event_addr_filters_sync()
3205 	if (event->addr_filters_gen != event->hw.addr_filters_gen) {  in perf_event_addr_filters_sync()
3206 		event->pmu->addr_filters_sync(event);  in perf_event_addr_filters_sync()
3207 		event->hw.addr_filters_gen = event->addr_filters_gen;  in perf_event_addr_filters_sync()
3209 	raw_spin_unlock(&ifh->lock);  in perf_event_addr_filters_sync()
3218 	if (event->attr.inherit || !is_sampling_event(event))  in _perf_event_refresh()
3219 		return -EINVAL;  in _perf_event_refresh()
3221 	atomic_add(refresh, &event->event_limit);  in _perf_event_refresh()
3252 	if (!bp->attr.disabled)  in perf_event_modify_breakpoint()
3259  * Copy event-type-independent attributes that may be modified.
3264 	to->sig_data = from->sig_data;  in perf_event_modify_copy_attr()
3274 	if (event->attr.type != attr->type)  in perf_event_modify_attr()
3275 		return -EINVAL;  in perf_event_modify_attr()
3277 	switch (event->attr.type) {  in perf_event_modify_attr()
3283 		return -EOPNOTSUPP;  in perf_event_modify_attr()
3286 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_event_modify_attr()
3288 	mutex_lock(&event->child_mutex);  in perf_event_modify_attr()
3290 	 * Event-type-independent attributes must be copied before event-type  in perf_event_modify_attr()
3294 	perf_event_modify_copy_attr(&event->attr, attr);  in perf_event_modify_attr()
3298 	list_for_each_entry(child, &event->child_list, child_list) {  in perf_event_modify_attr()
3299 		perf_event_modify_copy_attr(&child->attr, attr);  in perf_event_modify_attr()
3305 	mutex_unlock(&event->child_mutex);  in perf_event_modify_attr()
3312 	struct perf_event_context *ctx = pmu_ctx->ctx;  in __pmu_ctx_sched_out()
3314 	struct pmu *pmu = pmu_ctx->pmu;  in __pmu_ctx_sched_out()
3316 	if (ctx->task && !(ctx->is_active & EVENT_ALL)) {  in __pmu_ctx_sched_out()
3319 		cpc = this_cpu_ptr(pmu->cpu_pmu_context);  in __pmu_ctx_sched_out()
3320 		WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);  in __pmu_ctx_sched_out()
3321 		cpc->task_epc = NULL;  in __pmu_ctx_sched_out()
3330 					 &pmu_ctx->pinned_active,  in __pmu_ctx_sched_out()
3337 					 &pmu_ctx->flexible_active,  in __pmu_ctx_sched_out()
3345 		pmu_ctx->rotate_necessary = 0;  in __pmu_ctx_sched_out()
3356  * state, eg. ctx->is_active cleared even though most EPCs would still actually
3364 	int is_active = ctx->is_active;  in ctx_sched_out()
3369 	lockdep_assert_held(&ctx->lock);  in ctx_sched_out()
3371 	if (likely(!ctx->nr_events)) {  in ctx_sched_out()
3375 		WARN_ON_ONCE(ctx->is_active);  in ctx_sched_out()
3376 		if (ctx->task)  in ctx_sched_out()
3377 			WARN_ON_ONCE(cpuctx->task_ctx);  in ctx_sched_out()
3391 	__ctx_time_update(cpuctx, ctx, ctx == &cpuctx->ctx);  in ctx_sched_out()
3394 	 * CPU-release for the below ->is_active store,  in ctx_sched_out()
3398 	ctx->is_active &= ~event_type;  in ctx_sched_out()
3400 	if (!(ctx->is_active & EVENT_ALL)) {  in ctx_sched_out()
3405 		if (ctx->is_active & EVENT_FROZEN)  in ctx_sched_out()
3406 			ctx->is_active &= EVENT_TIME_FROZEN;  in ctx_sched_out()
3408 			ctx->is_active = 0;  in ctx_sched_out()
3411 	if (ctx->task) {  in ctx_sched_out()
3412 		WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in ctx_sched_out()
3413 		if (!(ctx->is_active & EVENT_ALL))  in ctx_sched_out()
3414 			cpuctx->task_ctx = NULL;  in ctx_sched_out()
3417 	is_active ^= ctx->is_active; /* changed bits */  in ctx_sched_out()
3434 	lockdep_assert_held(&ctx1->lock);  in context_equiv()
3435 	lockdep_assert_held(&ctx2->lock);  in context_equiv()
3438 	if (ctx1->pin_count || ctx2->pin_count)  in context_equiv()
3442 	if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)  in context_equiv()
3446 	if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)  in context_equiv()
3453 	if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&  in context_equiv()
3454 			ctx1->parent_gen == ctx2->parent_gen)  in context_equiv()
3466 	if (!event->attr.inherit_stat)  in __perf_event_sync_stat()
3476 	if (event->state == PERF_EVENT_STATE_ACTIVE)  in __perf_event_sync_stat()
3477 		event->pmu->read(event);  in __perf_event_sync_stat()
3482 	 * In order to keep per-task stats reliable we need to flip the event  in __perf_event_sync_stat()
3485 	value = local64_read(&next_event->count);  in __perf_event_sync_stat()
3486 	value = local64_xchg(&event->count, value);  in __perf_event_sync_stat()
3487 	local64_set(&next_event->count, value);  in __perf_event_sync_stat()
3489 	swap(event->total_time_enabled, next_event->total_time_enabled);  in __perf_event_sync_stat()
3490 	swap(event->total_time_running, next_event->total_time_running);  in __perf_event_sync_stat()
3504 	if (!ctx->nr_stat)  in perf_event_sync_stat()
3509 	event = list_first_entry(&ctx->event_list,  in perf_event_sync_stat()
3512 	next_event = list_first_entry(&next_ctx->event_list,  in perf_event_sync_stat()
3515 	while (&event->event_entry != &ctx->event_list &&  in perf_event_sync_stat()
3516 	       &next_event->event_entry != &next_ctx->event_list) {  in perf_event_sync_stat()
3538 	if (!prev_ctx->nr_task_data)  in perf_event_swap_task_ctx_data()
3542 				   &prev_ctx->pmu_ctx_list, &next_ctx->pmu_ctx_list,  in perf_event_swap_task_ctx_data()
3545 		if (WARN_ON_ONCE(prev_epc->pmu != next_epc->pmu))  in perf_event_swap_task_ctx_data()
3554 		if (prev_epc->pmu->swap_task_ctx)  in perf_event_swap_task_ctx_data()
3555 			prev_epc->pmu->swap_task_ctx(prev_epc, next_epc);  in perf_event_swap_task_ctx_data()
3557 			swap(prev_epc->task_ctx_data, next_epc->task_ctx_data);  in perf_event_swap_task_ctx_data()
3566 	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {  in perf_ctx_sched_task_cb()
3567 		cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);  in perf_ctx_sched_task_cb()
3569 		if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task)  in perf_ctx_sched_task_cb()
3570 			pmu_ctx->pmu->sched_task(pmu_ctx, sched_in);  in perf_ctx_sched_task_cb()
3577 	struct perf_event_context *ctx = task->perf_event_ctxp;  in perf_event_context_sched_out()
3586 	next_ctx = rcu_dereference(next->perf_event_ctxp);  in perf_event_context_sched_out()
3590 	parent = rcu_dereference(ctx->parent_ctx);  in perf_event_context_sched_out()
3591 	next_parent = rcu_dereference(next_ctx->parent_ctx);  in perf_event_context_sched_out()
3602 		 * lock (including re-checking that neither has been  in perf_event_context_sched_out()
3607 		raw_spin_lock(&ctx->lock);  in perf_event_context_sched_out()
3608 		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);  in perf_event_context_sched_out()
3613 			/* PMIs are disabled; ctx->nr_no_switch_fast is stable. */  in perf_event_context_sched_out()
3614 			if (local_read(&ctx->nr_no_switch_fast) ||  in perf_event_context_sched_out()
3615 			    local_read(&next_ctx->nr_no_switch_fast)) {  in perf_event_context_sched_out()
3618 				 * events that rely on the ctx->task relation.  in perf_event_context_sched_out()
3625 				raw_spin_unlock(&next_ctx->lock);  in perf_event_context_sched_out()
3630 			WRITE_ONCE(ctx->task, next);  in perf_event_context_sched_out()
3631 			WRITE_ONCE(next_ctx->task, task);  in perf_event_context_sched_out()
3641 			 * ctx->task and ctx->task_ctx_data are immaterial  in perf_event_context_sched_out()
3643 			 * ctx->lock which we're now holding.  in perf_event_context_sched_out()
3645 			RCU_INIT_POINTER(task->perf_event_ctxp, next_ctx);  in perf_event_context_sched_out()
3646 			RCU_INIT_POINTER(next->perf_event_ctxp, ctx);  in perf_event_context_sched_out()
3652 		raw_spin_unlock(&next_ctx->lock);  in perf_event_context_sched_out()
3653 		raw_spin_unlock(&ctx->lock);  in perf_event_context_sched_out()
3659 		raw_spin_lock(&ctx->lock);  in perf_event_context_sched_out()
3667 		raw_spin_unlock(&ctx->lock);  in perf_event_context_sched_out()
3676 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(pmu->cpu_pmu_context);  in perf_sched_cb_dec()
3681 	if (!--cpc->sched_cb_usage)  in perf_sched_cb_dec()
3682 		list_del(&cpc->sched_cb_entry);  in perf_sched_cb_dec()
3688 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(pmu->cpu_pmu_context);  in perf_sched_cb_inc()
3690 	if (!cpc->sched_cb_usage++)  in perf_sched_cb_inc()
3691 		list_add(&cpc->sched_cb_entry, this_cpu_ptr(&sched_cb_list));  in perf_sched_cb_inc()
3701  * This callback is relevant even to per-cpu events; for example multi event
3710 	pmu = cpc->epc.pmu;  in __perf_pmu_sched_task()
3713 	if (WARN_ON_ONCE(!pmu->sched_task))  in __perf_pmu_sched_task()
3716 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in __perf_pmu_sched_task()
3719 	pmu->sched_task(cpc->task_epc, sched_in);  in __perf_pmu_sched_task()
3722 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in __perf_pmu_sched_task()
3732 	/* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */  in perf_pmu_sched_task()
3733 	if (prev == next || cpuctx->task_ctx)  in perf_pmu_sched_task()
3747  * We stop each event and update the event value in event->count.
3768 	 * cgroup event are system-wide mode only  in __perf_event_task_sched_out()
3778 	return le->group_index < re->group_index;  in perf_less_group_idx()
3797 	struct perf_event **itrs = heap->data;  in __heap_add()
3800 		itrs[heap->nr] = event;  in __heap_add()
3801 		heap->nr++;  in __heap_add()
3809 	if (!pmu_ctx->ctx->task)  in __link_epc()
3812 	cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);  in __link_epc()
3813 	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);  in __link_epc()
3814 	cpc->task_epc = pmu_ctx;  in __link_epc()
3833 	if (pmu->filter && pmu->filter(pmu, cpu))  in visit_groups_merge()
3836 	if (!ctx->task) {  in visit_groups_merge()
3839 			.data = cpuctx->heap,  in visit_groups_merge()
3841 			.size = cpuctx->heap_size,  in visit_groups_merge()
3844 		lockdep_assert_held(&cpuctx->ctx.lock);  in visit_groups_merge()
3847 		if (cpuctx->cgrp)  in visit_groups_merge()
3848 			css = &cpuctx->cgrp->css;  in visit_groups_merge()
3857 		__heap_add(&event_heap, perf_event_groups_first(groups, -1, pmu, NULL));  in visit_groups_merge()
3864 	for (; css; css = css->parent)  in visit_groups_merge()
3865 		__heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, css->cgroup));  in visit_groups_merge()
3869 		__link_epc((*evt)->pmu_ctx);  in visit_groups_merge()
3870 		perf_assert_pmu_disabled((*evt)->pmu_ctx->pmu);  in visit_groups_merge()
3891  * Because the userpage is strictly per-event (there is no concept of context,
3893  * when context time starts :-(
3899 	if (likely(!atomic_read(&event->mmap_count)))  in event_update_userpage()
3921 	struct perf_event_context *ctx = event->ctx;  in merge_sched_in()
3924 	if (event->state <= PERF_EVENT_STATE_OFF)  in merge_sched_in()
3932 			list_add_tail(&event->active_list, get_event_list(event));  in merge_sched_in()
3935 	if (event->state == PERF_EVENT_STATE_INACTIVE) {  in merge_sched_in()
3937 		if (event->attr.pinned) {  in merge_sched_in()
3943 			event->pmu_ctx->rotate_necessary = 1;  in merge_sched_in()
3944 			cpc = this_cpu_ptr(event->pmu_ctx->pmu->cpu_pmu_context);  in merge_sched_in()
3965 	struct perf_event_context *ctx = pmu_ctx->ctx;  in __pmu_ctx_sched_in()
3968 		pmu_groups_sched_in(ctx, &ctx->pinned_groups, pmu_ctx->pmu);  in __pmu_ctx_sched_in()
3970 		pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu_ctx->pmu);  in __pmu_ctx_sched_in()
3978 	int is_active = ctx->is_active;  in ctx_sched_in()
3983 	lockdep_assert_held(&ctx->lock);  in ctx_sched_in()
3985 	if (likely(!ctx->nr_events))  in ctx_sched_in()
3993 		 * CPU-release for the below ->is_active store,  in ctx_sched_in()
3999 	ctx->is_active |= (event_type | EVENT_TIME);  in ctx_sched_in()
4000 	if (ctx->task) {  in ctx_sched_in()
4002 			cpuctx->task_ctx = ctx;  in ctx_sched_in()
4004 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in ctx_sched_in()
4007 	is_active ^= ctx->is_active; /* changed bits */  in ctx_sched_in()
4031 	ctx = rcu_dereference(task->perf_event_ctxp);  in perf_event_context_sched_in()
4035 	if (cpuctx->task_ctx == ctx) {  in perf_event_context_sched_in()
4048 	 * We must check ctx->nr_events while holding ctx->lock, such  in perf_event_context_sched_in()
4051 	if (!ctx->nr_events)  in perf_event_context_sched_in()
4063 	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) {  in perf_event_context_sched_in()
4064 		perf_ctx_disable(&cpuctx->ctx, false);  in perf_event_context_sched_in()
4065 		ctx_sched_out(&cpuctx->ctx, NULL, EVENT_FLEXIBLE);  in perf_event_context_sched_in()
4070 	perf_ctx_sched_task_cb(cpuctx->task_ctx, true);  in perf_event_context_sched_in()
4072 	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))  in perf_event_context_sched_in()
4073 		perf_ctx_enable(&cpuctx->ctx, false);  in perf_event_context_sched_in()
4108 	u64 frequency = event->attr.sample_freq;  in perf_calculate_period()
4124 	 * period = -------------------  in perf_calculate_period()
4137 		a##_fls--;		\  in perf_calculate_period()
4140 		b##_fls--;		\  in perf_calculate_period()
4184 	struct hw_perf_event *hwc = &event->hw;  in perf_adjust_period()
4190 	delta = (s64)(period - hwc->sample_period);  in perf_adjust_period()
4194 		delta -= 7;  in perf_adjust_period()
4197 	sample_period = hwc->sample_period + delta;  in perf_adjust_period()
4202 	hwc->sample_period = sample_period;  in perf_adjust_period()
4204 	if (local64_read(&hwc->period_left) > 8*sample_period) {  in perf_adjust_period()
4206 			event->pmu->stop(event, PERF_EF_UPDATE);  in perf_adjust_period()
4208 		local64_set(&hwc->period_left, 0);  in perf_adjust_period()
4211 			event->pmu->start(event, PERF_EF_RELOAD);  in perf_adjust_period()
4223 		if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_adjust_freq_unthr_events()
4226 		// XXX use visit thingy to avoid the -1,cpu match  in perf_adjust_freq_unthr_events()
4230 		hwc = &event->hw;  in perf_adjust_freq_unthr_events()
4232 		if (hwc->interrupts == MAX_INTERRUPTS) {  in perf_adjust_freq_unthr_events()
4233 			hwc->interrupts = 0;  in perf_adjust_freq_unthr_events()
4235 			if (!event->attr.freq || !event->attr.sample_freq)  in perf_adjust_freq_unthr_events()
4236 				event->pmu->start(event, 0);  in perf_adjust_freq_unthr_events()
4239 		if (!event->attr.freq || !event->attr.sample_freq)  in perf_adjust_freq_unthr_events()
4243 		 * stop the event and update event->count  in perf_adjust_freq_unthr_events()
4245 		event->pmu->stop(event, PERF_EF_UPDATE);  in perf_adjust_freq_unthr_events()
4247 		now = local64_read(&event->count);  in perf_adjust_freq_unthr_events()
4248 		delta = now - hwc->freq_count_stamp;  in perf_adjust_freq_unthr_events()
4249 		hwc->freq_count_stamp = now;  in perf_adjust_freq_unthr_events()
4261 		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);  in perf_adjust_freq_unthr_events()
4277 	 * - context have events in frequency mode (needs freq adjust)  in perf_adjust_freq_unthr_context()
4278 	 * - there are events to unthrottle on this cpu  in perf_adjust_freq_unthr_context()
4280 	if (!(ctx->nr_freq || unthrottle))  in perf_adjust_freq_unthr_context()
4283 	raw_spin_lock(&ctx->lock);  in perf_adjust_freq_unthr_context()
4285 	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {  in perf_adjust_freq_unthr_context()
4286 		if (!(pmu_ctx->nr_freq || unthrottle))  in perf_adjust_freq_unthr_context()
4290 		if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT)  in perf_adjust_freq_unthr_context()
4293 		perf_pmu_disable(pmu_ctx->pmu);  in perf_adjust_freq_unthr_context()
4294 		perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active);  in perf_adjust_freq_unthr_context()
4295 		perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active);  in perf_adjust_freq_unthr_context()
4296 		perf_pmu_enable(pmu_ctx->pmu);  in perf_adjust_freq_unthr_context()
4299 	raw_spin_unlock(&ctx->lock);  in perf_adjust_freq_unthr_context()
4308 	 * Rotate the first entry last of non-pinned groups. Rotation might be  in rotate_ctx()
4311 	if (ctx->rotate_disable)  in rotate_ctx()
4314 	perf_event_groups_delete(&ctx->flexible_groups, event);  in rotate_ctx()
4315 	perf_event_groups_insert(&ctx->flexible_groups, event);  in rotate_ctx()
4326 		.pmu = pmu_ctx->pmu,  in ctx_event_to_rotate()
4330 	event = list_first_entry_or_null(&pmu_ctx->flexible_active,  in ctx_event_to_rotate()
4336 	tree = &pmu_ctx->ctx->flexible_groups.tree;  in ctx_event_to_rotate()
4338 	if (!pmu_ctx->ctx->task) {  in ctx_event_to_rotate()
4347 	key.cpu = -1;  in ctx_event_to_rotate()
4364 	pmu_ctx->rotate_necessary = 0;  in ctx_event_to_rotate()
4382 	cpu_epc = &cpc->epc;  in perf_rotate_context()
4383 	pmu = cpu_epc->pmu;  in perf_rotate_context()
4384 	task_epc = cpc->task_epc;  in perf_rotate_context()
4386 	cpu_rotate = cpu_epc->rotate_necessary;  in perf_rotate_context()
4387 	task_rotate = task_epc ? task_epc->rotate_necessary : 0;  in perf_rotate_context()
4392 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_rotate_context()
4405 		update_context_time(task_epc->ctx);  in perf_rotate_context()
4410 		update_context_time(&cpuctx->ctx);  in perf_rotate_context()
4412 		rotate_ctx(&cpuctx->ctx, cpu_event);  in perf_rotate_context()
4417 		rotate_ctx(task_epc->ctx, task_event);  in perf_rotate_context()
4423 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_rotate_context()
4440 	perf_adjust_freq_unthr_context(&cpuctx->ctx, !!throttled);  in perf_event_task_tick()
4443 	ctx = rcu_dereference(current->perf_event_ctxp);  in perf_event_task_tick()
4452 	if (!event->attr.enable_on_exec)  in event_enable_on_exec()
4455 	event->attr.enable_on_exec = 0;  in event_enable_on_exec()
4456 	if (event->state >= PERF_EVENT_STATE_INACTIVE)  in event_enable_on_exec()
4465  * Enable all of a task's events that have been marked enable-on-exec.
4478 	if (WARN_ON_ONCE(current->perf_event_ctxp != ctx))  in perf_event_enable_on_exec()
4481 	if (!ctx->nr_events)  in perf_event_enable_on_exec()
4488 	list_for_each_entry(event, &ctx->event_list, event_entry) {  in perf_event_enable_on_exec()
4515  * remove-on-exec, and feeds their values back to parent events.
4524 	mutex_lock(&ctx->mutex);  in perf_event_remove_on_exec()
4526 	if (WARN_ON_ONCE(ctx->task != current))  in perf_event_remove_on_exec()
4529 	list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) {  in perf_event_remove_on_exec()
4530 		if (!event->attr.remove_on_exec)  in perf_event_remove_on_exec()
4541 	raw_spin_lock_irqsave(&ctx->lock, flags);  in perf_event_remove_on_exec()
4544 	raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_event_remove_on_exec()
4547 	mutex_unlock(&ctx->mutex);  in perf_event_remove_on_exec()
4569 	if (event->group_caps & PERF_EV_CAP_READ_SCOPE) {  in __perf_event_read_cpu()
4570 		const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(event->pmu->scope, event_cpu);  in __perf_event_read_cpu()
4576 	if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {  in __perf_event_read_cpu()
4593 	struct perf_event *sub, *event = data->event;  in __perf_event_read()
4594 	struct perf_event_context *ctx = event->ctx;  in __perf_event_read()
4596 	struct pmu *pmu = event->pmu;  in __perf_event_read()
4602 	 * event->count would have been updated to a recent sample  in __perf_event_read()
4605 	if (ctx->task && cpuctx->task_ctx != ctx)  in __perf_event_read()
4608 	raw_spin_lock(&ctx->lock);  in __perf_event_read()
4612 	if (data->group)  in __perf_event_read()
4615 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in __perf_event_read()
4618 	if (!data->group) {  in __perf_event_read()
4619 		pmu->read(event);  in __perf_event_read()
4620 		data->ret = 0;  in __perf_event_read()
4624 	pmu->start_txn(pmu, PERF_PMU_TXN_READ);  in __perf_event_read()
4626 	pmu->read(event);  in __perf_event_read()
4629 		if (sub->state == PERF_EVENT_STATE_ACTIVE) {  in __perf_event_read()
4634 			sub->pmu->read(sub);  in __perf_event_read()
4638 	data->ret = pmu->commit_txn(pmu);  in __perf_event_read()
4641 	raw_spin_unlock(&ctx->lock);  in __perf_event_read()
4647 		return local64_read(&event->count);  in perf_event_count()
4649 	return local64_read(&event->count) + atomic64_read(&event->child_count);  in perf_event_count()
4665  * NMI-safe method to read a local event, that is an event that
4667  *   - either for the current task, or for this CPU
4668  *   - does not have inherit set, for inherited task events
4670  *   - must not have a pmu::count method
4690 	if (event->attr.inherit) {  in perf_event_read_local()
4691 		ret = -EOPNOTSUPP;  in perf_event_read_local()
4695 	/* If this is a per-task event, it must be for current */  in perf_event_read_local()
4696 	if ((event->attach_state & PERF_ATTACH_TASK) &&  in perf_event_read_local()
4697 	    event->hw.target != current) {  in perf_event_read_local()
4698 		ret = -EINVAL;  in perf_event_read_local()
4704 	 * a per-package event that can be read locally  in perf_event_read_local()
4706 	event_oncpu = __perf_event_read_cpu(event, event->oncpu);  in perf_event_read_local()
4707 	event_cpu = __perf_event_read_cpu(event, event->cpu);  in perf_event_read_local()
4709 	/* If this is a per-CPU event, it must be for this CPU */  in perf_event_read_local()
4710 	if (!(event->attach_state & PERF_ATTACH_TASK) &&  in perf_event_read_local()
4712 		ret = -EINVAL;  in perf_event_read_local()
4717 	if (event->attr.pinned && event_oncpu != smp_processor_id()) {  in perf_event_read_local()
4718 		ret = -EBUSY;  in perf_event_read_local()
4723 	 * If the event is currently on this CPU, its either a per-task event,  in perf_event_read_local()
4725 	 * oncpu == -1).  in perf_event_read_local()
4728 		event->pmu->read(event);  in perf_event_read_local()
4730 	*value = local64_read(&event->count);  in perf_event_read_local()
4748 	enum perf_event_state state = READ_ONCE(event->state);  in perf_event_read()
4760 		 * Orders the ->state and ->oncpu loads such that if we see  in perf_event_read()
4761 		 * ACTIVE we must also see the right ->oncpu.  in perf_event_read()
4767 		event_cpu = READ_ONCE(event->oncpu);  in perf_event_read()
4787 		 * Therefore, either way, we'll have an up-to-date event count  in perf_event_read()
4795 		struct perf_event_context *ctx = event->ctx;  in perf_event_read()
4798 		raw_spin_lock_irqsave(&ctx->lock, flags);  in perf_event_read()
4799 		state = event->state;  in perf_event_read()
4801 			raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_event_read()
4814 		raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_event_read()
4825 	raw_spin_lock_init(&ctx->lock);  in __perf_event_init_context()
4826 	mutex_init(&ctx->mutex);  in __perf_event_init_context()
4827 	INIT_LIST_HEAD(&ctx->pmu_ctx_list);  in __perf_event_init_context()
4828 	perf_event_groups_init(&ctx->pinned_groups);  in __perf_event_init_context()
4829 	perf_event_groups_init(&ctx->flexible_groups);  in __perf_event_init_context()
4830 	INIT_LIST_HEAD(&ctx->event_list);  in __perf_event_init_context()
4831 	refcount_set(&ctx->refcount, 1);  in __perf_event_init_context()
4837 	epc->pmu = pmu;  in __perf_init_event_pmu_context()
4838 	INIT_LIST_HEAD(&epc->pmu_ctx_entry);  in __perf_init_event_pmu_context()
4839 	INIT_LIST_HEAD(&epc->pinned_active);  in __perf_init_event_pmu_context()
4840 	INIT_LIST_HEAD(&epc->flexible_active);  in __perf_init_event_pmu_context()
4841 	atomic_set(&epc->refcount, 1);  in __perf_init_event_pmu_context()
4855 		ctx->task = get_task_struct(task);  in alloc_perf_context()
4875 		return ERR_PTR(-ESRCH);  in find_lively_task_by_vpid()
4893 		err = perf_allow_cpu(&event->attr);  in find_get_context()
4897 		cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);  in find_get_context()
4898 		ctx = &cpuctx->ctx;  in find_get_context()
4900 		raw_spin_lock_irqsave(&ctx->lock, flags);  in find_get_context()
4901 		++ctx->pin_count;  in find_get_context()
4902 		raw_spin_unlock_irqrestore(&ctx->lock, flags);  in find_get_context()
4907 	err = -EINVAL;  in find_get_context()
4912 		++ctx->pin_count;  in find_get_context()
4914 		raw_spin_unlock_irqrestore(&ctx->lock, flags);  in find_get_context()
4920 		err = -ENOMEM;  in find_get_context()
4925 		mutex_lock(&task->perf_event_mutex);  in find_get_context()
4930 		if (task->flags & PF_EXITING)  in find_get_context()
4931 			err = -ESRCH;  in find_get_context()
4932 		else if (task->perf_event_ctxp)  in find_get_context()
4933 			err = -EAGAIN;  in find_get_context()
4936 			++ctx->pin_count;  in find_get_context()
4937 			rcu_assign_pointer(task->perf_event_ctxp, ctx);  in find_get_context()
4939 		mutex_unlock(&task->perf_event_mutex);  in find_get_context()
4944 			if (err == -EAGAIN)  in find_get_context()
4963 	if (!ctx->task) {  in find_get_pmu_context()
4971 		cpc = per_cpu_ptr(pmu->cpu_pmu_context, event->cpu);  in find_get_pmu_context()
4972 		epc = &cpc->epc;  in find_get_pmu_context()
4973 		raw_spin_lock_irq(&ctx->lock);  in find_get_pmu_context()
4974 		if (!epc->ctx) {  in find_get_pmu_context()
4975 			atomic_set(&epc->refcount, 1);  in find_get_pmu_context()
4976 			epc->embedded = 1;  in find_get_pmu_context()
4977 			list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);  in find_get_pmu_context()
4978 			epc->ctx = ctx;  in find_get_pmu_context()
4980 			WARN_ON_ONCE(epc->ctx != ctx);  in find_get_pmu_context()
4981 			atomic_inc(&epc->refcount);  in find_get_pmu_context()
4983 		raw_spin_unlock_irq(&ctx->lock);  in find_get_pmu_context()
4989 		return ERR_PTR(-ENOMEM);  in find_get_pmu_context()
4991 	if (event->attach_state & PERF_ATTACH_TASK_DATA) {  in find_get_pmu_context()
4995 			return ERR_PTR(-ENOMEM);  in find_get_pmu_context()
5004 	 * lockdep_assert_held(&ctx->mutex);  in find_get_pmu_context()
5007 	 * child_ctx->mutex.  in find_get_pmu_context()
5010 	raw_spin_lock_irq(&ctx->lock);  in find_get_pmu_context()
5011 	list_for_each_entry(epc, &ctx->pmu_ctx_list, pmu_ctx_entry) {  in find_get_pmu_context()
5012 		if (epc->pmu == pmu) {  in find_get_pmu_context()
5013 			WARN_ON_ONCE(epc->ctx != ctx);  in find_get_pmu_context()
5014 			atomic_inc(&epc->refcount);  in find_get_pmu_context()
5022 	list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);  in find_get_pmu_context()
5023 	epc->ctx = ctx;  in find_get_pmu_context()
5026 	if (task_ctx_data && !epc->task_ctx_data) {  in find_get_pmu_context()
5027 		epc->task_ctx_data = task_ctx_data;  in find_get_pmu_context()
5029 		ctx->nr_task_data++;  in find_get_pmu_context()
5031 	raw_spin_unlock_irq(&ctx->lock);  in find_get_pmu_context()
5041 	WARN_ON_ONCE(!atomic_inc_not_zero(&epc->refcount));  in get_pmu_ctx()
5048 	kfree(epc->task_ctx_data);  in free_epc_rcu()
5054 	struct perf_event_context *ctx = epc->ctx;  in put_pmu_ctx()
5060 	 * lockdep_assert_held(&ctx->mutex);  in put_pmu_ctx()
5062 	 * can't because of the call-site in _free_event()/put_event()  in put_pmu_ctx()
5063 	 * which isn't always called under ctx->mutex.  in put_pmu_ctx()
5065 	if (!atomic_dec_and_raw_lock_irqsave(&epc->refcount, &ctx->lock, flags))  in put_pmu_ctx()
5068 	WARN_ON_ONCE(list_empty(&epc->pmu_ctx_entry));  in put_pmu_ctx()
5070 	list_del_init(&epc->pmu_ctx_entry);  in put_pmu_ctx()
5071 	epc->ctx = NULL;  in put_pmu_ctx()
5073 	WARN_ON_ONCE(!list_empty(&epc->pinned_active));  in put_pmu_ctx()
5074 	WARN_ON_ONCE(!list_empty(&epc->flexible_active));  in put_pmu_ctx()
5076 	raw_spin_unlock_irqrestore(&ctx->lock, flags);  in put_pmu_ctx()
5078 	if (epc->embedded)  in put_pmu_ctx()
5081 	call_rcu(&epc->rcu_head, free_epc_rcu);  in put_pmu_ctx()
5090 	if (event->ns)  in free_event_rcu()
5091 		put_pid_ns(event->ns);  in free_event_rcu()
5101 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);  in detach_sb_event()
5103 	raw_spin_lock(&pel->lock);  in detach_sb_event()
5104 	list_del_rcu(&event->sb_list);  in detach_sb_event()
5105 	raw_spin_unlock(&pel->lock);  in detach_sb_event()
5110 	struct perf_event_attr *attr = &event->attr;  in is_sb_event()
5112 	if (event->parent)  in is_sb_event()
5115 	if (event->attach_state & PERF_ATTACH_TASK)  in is_sb_event()
5118 	if (attr->mmap || attr->mmap_data || attr->mmap2 ||  in is_sb_event()
5119 	    attr->comm || attr->comm_exec ||  in is_sb_event()
5120 	    attr->task || attr->ksymbol ||  in is_sb_event()
5121 	    attr->context_switch || attr->text_poke ||  in is_sb_event()
5122 	    attr->bpf_event)  in is_sb_event()
5159 	if (event->parent)  in unaccount_event()
5162 	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))  in unaccount_event()
5164 	if (event->attr.mmap || event->attr.mmap_data)  in unaccount_event()
5166 	if (event->attr.build_id)  in unaccount_event()
5168 	if (event->attr.comm)  in unaccount_event()
5170 	if (event->attr.namespaces)  in unaccount_event()
5172 	if (event->attr.cgroup)  in unaccount_event()
5174 	if (event->attr.task)  in unaccount_event()
5176 	if (event->attr.freq)  in unaccount_event()
5178 	if (event->attr.context_switch) {  in unaccount_event()
5186 	if (event->attr.ksymbol)  in unaccount_event()
5188 	if (event->attr.bpf_event)  in unaccount_event()
5190 	if (event->attr.text_poke)  in unaccount_event()
5194 		if (!atomic_add_unless(&perf_sched_count, -1, 1))  in unaccount_event()
5214  *  1) cpu-wide events in the presence of per-task events,
5215  *  2) per-task events in the presence of cpu-wide events,
5219  * _free_event()), the latter -- before the first perf_install_in_context().
5223 	struct pmu *pmu = event->pmu;  in exclusive_event_init()
5229 	 * Prevent co-existence of per-task and cpu-wide events on the  in exclusive_event_init()
5232 	 * Negative pmu::exclusive_cnt means there are cpu-wide  in exclusive_event_init()
5234 	 * per-task events.  in exclusive_event_init()
5238 	 * to mean "per-task event", because unlike other attach states it  in exclusive_event_init()
5241 	if (event->attach_state & PERF_ATTACH_TASK) {  in exclusive_event_init()
5242 		if (!atomic_inc_unless_negative(&pmu->exclusive_cnt))  in exclusive_event_init()
5243 			return -EBUSY;  in exclusive_event_init()
5245 		if (!atomic_dec_unless_positive(&pmu->exclusive_cnt))  in exclusive_event_init()
5246 			return -EBUSY;  in exclusive_event_init()
5254 	struct pmu *pmu = event->pmu;  in exclusive_event_destroy()
5260 	if (event->attach_state & PERF_ATTACH_TASK)  in exclusive_event_destroy()
5261 		atomic_dec(&pmu->exclusive_cnt);  in exclusive_event_destroy()
5263 		atomic_inc(&pmu->exclusive_cnt);  in exclusive_event_destroy()
5268 	if ((e1->pmu == e2->pmu) &&  in exclusive_event_match()
5269 	    (e1->cpu == e2->cpu ||  in exclusive_event_match()
5270 	     e1->cpu == -1 ||  in exclusive_event_match()
5271 	     e2->cpu == -1))  in exclusive_event_match()
5280 	struct pmu *pmu = event->pmu;  in exclusive_event_installable()
5282 	lockdep_assert_held(&ctx->mutex);  in exclusive_event_installable()
5287 	list_for_each_entry(iter_event, &ctx->event_list, event_entry) {  in exclusive_event_installable()
5300 	struct callback_head *head = &event->pending_task;  in perf_pending_task_sync()
5302 	if (!event->pending_work)  in perf_pending_task_sync()
5309 		event->pending_work = 0;  in perf_pending_task_sync()
5310 		local_dec(&event->ctx->nr_no_switch_fast);  in perf_pending_task_sync()
5319 	rcuwait_wait_event(&event->pending_work_wait, !event->pending_work, TASK_UNINTERRUPTIBLE);  in perf_pending_task_sync()
5324 	irq_work_sync(&event->pending_irq);  in _free_event()
5325 	irq_work_sync(&event->pending_disable_irq);  in _free_event()
5332 	if (event->rb) {  in _free_event()
5334 		 * Can happen when we close an event with re-directed output.  in _free_event()
5339 		mutex_lock(&event->mmap_mutex);  in _free_event()
5341 		mutex_unlock(&event->mmap_mutex);  in _free_event()
5347 	if (!event->parent) {  in _free_event()
5348 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)  in _free_event()
5354 	kfree(event->addr_filter_ranges);  in _free_event()
5356 	if (event->destroy)  in _free_event()
5357 		event->destroy(event);  in _free_event()
5360 	 * Must be after ->destroy(), due to uprobe_perf_close() using  in _free_event()
5363 	if (event->hw.target)  in _free_event()
5364 		put_task_struct(event->hw.target);  in _free_event()
5366 	if (event->pmu_ctx)  in _free_event()
5367 		put_pmu_ctx(event->pmu_ctx);  in _free_event()
5373 	if (event->ctx)  in _free_event()
5374 		put_ctx(event->ctx);  in _free_event()
5377 	module_put(event->pmu->module);  in _free_event()
5379 	call_rcu(&event->rcu_head, free_event_rcu);  in _free_event()
5388 	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,  in free_event()
5390 				atomic_long_read(&event->refcount), event)) {  in free_event()
5391 		/* leak to avoid use-after-free */  in free_event()
5410 	 * owner->perf_event_mutex.  in perf_remove_from_owner()
5412 	owner = READ_ONCE(event->owner);  in perf_remove_from_owner()
5426 		 * holding ctx->mutex which would be an inversion wrt. the  in perf_remove_from_owner()
5430 		 * ctx->mutex.  in perf_remove_from_owner()
5432 		mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING);  in perf_remove_from_owner()
5435 		 * We have to re-check the event->owner field, if it is cleared  in perf_remove_from_owner()
5440 		if (event->owner) {  in perf_remove_from_owner()
5441 			list_del_init(&event->owner_entry);  in perf_remove_from_owner()
5442 			smp_store_release(&event->owner, NULL);  in perf_remove_from_owner()
5444 		mutex_unlock(&owner->perf_event_mutex);  in perf_remove_from_owner()
5451 	if (!atomic_long_dec_and_test(&event->refcount))  in put_event()
5464 	struct perf_event_context *ctx = event->ctx;  in perf_event_release_kernel()
5473 		WARN_ON_ONCE(event->attach_state &  in perf_event_release_kernel()
5482 	WARN_ON_ONCE(ctx->parent_ctx);  in perf_event_release_kernel()
5488 	 * Anybody acquiring event->child_mutex after the below loop _must_  in perf_event_release_kernel()
5500 	mutex_lock(&event->child_mutex);  in perf_event_release_kernel()
5501 	list_for_each_entry(child, &event->child_list, child_list) {  in perf_event_release_kernel()
5508 		ctx = READ_ONCE(child->ctx);  in perf_event_release_kernel()
5522 		 * can re-acquire child_mutex.  in perf_event_release_kernel()
5524 		mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
5525 		mutex_lock(&ctx->mutex);  in perf_event_release_kernel()
5526 		mutex_lock(&event->child_mutex);  in perf_event_release_kernel()
5533 		tmp = list_first_entry_or_null(&event->child_list,  in perf_event_release_kernel()
5537 			list_move(&child->child_list, &free_list);  in perf_event_release_kernel()
5544 			var = &ctx->refcount;  in perf_event_release_kernel()
5547 		mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
5548 		mutex_unlock(&ctx->mutex);  in perf_event_release_kernel()
5562 	mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
5565 		void *var = &child->ctx->refcount;  in perf_event_release_kernel()
5567 		list_del(&child->child_list);  in perf_event_release_kernel()
5589 	perf_event_release_kernel(file->private_data);  in perf_release()
5601 	mutex_lock(&event->child_mutex);  in __perf_event_read_value()
5606 	*enabled += event->total_time_enabled +  in __perf_event_read_value()
5607 			atomic64_read(&event->child_total_time_enabled);  in __perf_event_read_value()
5608 	*running += event->total_time_running +  in __perf_event_read_value()
5609 			atomic64_read(&event->child_total_time_running);  in __perf_event_read_value()
5611 	list_for_each_entry(child, &event->child_list, child_list) {  in __perf_event_read_value()
5614 		*enabled += child->total_time_enabled;  in __perf_event_read_value()
5615 		*running += child->total_time_running;  in __perf_event_read_value()
5617 	mutex_unlock(&event->child_mutex);  in __perf_event_read_value()
5638 	struct perf_event_context *ctx = leader->ctx;  in __perf_read_group_add()
5648 	raw_spin_lock_irqsave(&ctx->lock, flags);  in __perf_read_group_add()
5654 	 *  - leader->ctx->lock pins leader->sibling_list  in __perf_read_group_add()
5655 	 *  - parent->child_mutex pins parent->child_list  in __perf_read_group_add()
5656 	 *  - parent->ctx->mutex pins parent->sibling_list  in __perf_read_group_add()
5658 	 * Because parent->ctx != leader->ctx (and child_list nests inside  in __perf_read_group_add()
5659 	 * ctx->mutex), group destruction is not atomic between children, also  in __perf_read_group_add()
5669 	parent = leader->parent;  in __perf_read_group_add()
5671 	    (parent->group_generation != leader->group_generation ||  in __perf_read_group_add()
5672 	     parent->nr_siblings != leader->nr_siblings)) {  in __perf_read_group_add()
5673 		ret = -ECHILD;  in __perf_read_group_add()
5678 	 * Since we co-schedule groups, {enabled,running} times of siblings  in __perf_read_group_add()
5683 		values[n++] += leader->total_time_enabled +  in __perf_read_group_add()
5684 			atomic64_read(&leader->child_total_time_enabled);  in __perf_read_group_add()
5688 		values[n++] += leader->total_time_running +  in __perf_read_group_add()
5689 			atomic64_read(&leader->child_total_time_running);  in __perf_read_group_add()
5699 		values[n++] = atomic64_read(&leader->lost_samples);  in __perf_read_group_add()
5706 			values[n++] = atomic64_read(&sub->lost_samples);  in __perf_read_group_add()
5710 	raw_spin_unlock_irqrestore(&ctx->lock, flags);  in __perf_read_group_add()
5717 	struct perf_event *leader = event->group_leader, *child;  in perf_read_group()
5718 	struct perf_event_context *ctx = leader->ctx;  in perf_read_group()
5722 	lockdep_assert_held(&ctx->mutex);  in perf_read_group()
5724 	values = kzalloc(event->read_size, GFP_KERNEL);  in perf_read_group()
5726 		return -ENOMEM;  in perf_read_group()
5728 	values[0] = 1 + leader->nr_siblings;  in perf_read_group()
5730 	mutex_lock(&leader->child_mutex);  in perf_read_group()
5736 	list_for_each_entry(child, &leader->child_list, child_list) {  in perf_read_group()
5742 	mutex_unlock(&leader->child_mutex);  in perf_read_group()
5744 	ret = event->read_size;  in perf_read_group()
5745 	if (copy_to_user(buf, values, event->read_size))  in perf_read_group()
5746 		ret = -EFAULT;  in perf_read_group()
5750 	mutex_unlock(&leader->child_mutex);  in perf_read_group()
5771 		values[n++] = atomic64_read(&event->lost_samples);  in perf_read_one()
5774 		return -EFAULT;  in perf_read_one()
5783 	if (event->state > PERF_EVENT_STATE_EXIT)  in is_event_hup()
5786 	mutex_lock(&event->child_mutex);  in is_event_hup()
5787 	no_children = list_empty(&event->child_list);  in is_event_hup()
5788 	mutex_unlock(&event->child_mutex);  in is_event_hup()
5793  * Read the performance event - simple non blocking version for now
5798 	u64 read_format = event->attr.read_format;  in __perf_read()
5802 	 * Return end-of-file for a read on an event that is in  in __perf_read()
5806 	if (event->state == PERF_EVENT_STATE_ERROR)  in __perf_read()
5809 	if (count < event->read_size)  in __perf_read()
5810 		return -ENOSPC;  in __perf_read()
5812 	WARN_ON_ONCE(event->ctx->parent_ctx);  in __perf_read()
5824 	struct perf_event *event = file->private_data;  in perf_read()
5841 	struct perf_event *event = file->private_data;  in perf_poll()
5845 	poll_wait(file, &event->waitq, wait);  in perf_poll()
5851 	 * Pin the event->rb by taking event->mmap_mutex; otherwise  in perf_poll()
5854 	mutex_lock(&event->mmap_mutex);  in perf_poll()
5855 	rb = event->rb;  in perf_poll()
5857 		events = atomic_xchg(&rb->poll, 0);  in perf_poll()
5858 	mutex_unlock(&event->mmap_mutex);  in perf_poll()
5865 	local64_set(&event->count, 0);  in _perf_event_reset()
5876 	WARN_ON_ONCE(event->attr.inherit);  in perf_event_pause()
5878 	count = local64_read(&event->count);  in perf_event_pause()
5880 		local64_set(&event->count, 0);  in perf_event_pause()
5888  * Holding the top-level event's child_mutex means that any
5898 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_event_for_each_child()
5900 	mutex_lock(&event->child_mutex);  in perf_event_for_each_child()
5902 	list_for_each_entry(child, &event->child_list, child_list)  in perf_event_for_each_child()
5904 	mutex_unlock(&event->child_mutex);  in perf_event_for_each_child()
5910 	struct perf_event_context *ctx = event->ctx;  in perf_event_for_each()
5913 	lockdep_assert_held(&ctx->mutex);  in perf_event_for_each()
5915 	event = event->group_leader;  in perf_event_for_each()
5930 	if (event->attr.freq) {  in __perf_event_period()
5931 		event->attr.sample_freq = value;  in __perf_event_period()
5933 		event->attr.sample_period = value;  in __perf_event_period()
5934 		event->hw.sample_period = value;  in __perf_event_period()
5937 	active = (event->state == PERF_EVENT_STATE_ACTIVE);  in __perf_event_period()
5939 		perf_pmu_disable(event->pmu);  in __perf_event_period()
5942 		 * trying to unthrottle while we already re-started the event.  in __perf_event_period()
5944 		if (event->hw.interrupts == MAX_INTERRUPTS) {  in __perf_event_period()
5945 			event->hw.interrupts = 0;  in __perf_event_period()
5948 		event->pmu->stop(event, PERF_EF_UPDATE);  in __perf_event_period()
5951 	local64_set(&event->hw.period_left, 0);  in __perf_event_period()
5954 		event->pmu->start(event, PERF_EF_RELOAD);  in __perf_event_period()
5955 		perf_pmu_enable(event->pmu);  in __perf_event_period()
5961 	return event->pmu->check_period(event, value);  in perf_event_check_period()
5967 		return -EINVAL;  in _perf_event_period()
5970 		return -EINVAL;  in _perf_event_period()
5972 	if (event->attr.freq && value > sysctl_perf_event_sample_rate)  in _perf_event_period()
5973 		return -EINVAL;  in _perf_event_period()
5976 		return -EINVAL;  in _perf_event_period()
5978 	if (!event->attr.freq && (value & (1ULL << 63)))  in _perf_event_period()
5979 		return -EINVAL;  in _perf_event_period()
6005 		return -EBADF;  in perf_fget_light()
6007 	if (fd_file(f)->f_op != &perf_fops) {  in perf_fget_light()
6009 		return -EBADF;  in perf_fget_light()
6045 			return -EFAULT;  in _perf_ioctl()
6054 			return -EFAULT;  in _perf_ioctl()
6061 		if (arg != -1) {  in _perf_ioctl()
6067 			output_event = fd_file(output)->private_data;  in _perf_ioctl()
6101 		rb = rcu_dereference(event->rb);  in _perf_ioctl()
6102 		if (!rb || !rb->nr_pages) {  in _perf_ioctl()
6104 			return -EINVAL;  in _perf_ioctl()
6125 		return -ENOTTY;  in _perf_ioctl()
6138 	struct perf_event *event = file->private_data;  in perf_ioctl()
6163 		/* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */  in perf_compat_ioctl()
6181 	mutex_lock(&current->perf_event_mutex);  in perf_event_task_enable()
6182 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {  in perf_event_task_enable()
6187 	mutex_unlock(&current->perf_event_mutex);  in perf_event_task_enable()
6197 	mutex_lock(&current->perf_event_mutex);  in perf_event_task_disable()
6198 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {  in perf_event_task_disable()
6203 	mutex_unlock(&current->perf_event_mutex);  in perf_event_task_disable()
6210 	if (event->hw.state & PERF_HES_STOPPED)  in perf_event_index()
6213 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_event_index()
6216 	return event->pmu->event_idx(event);  in perf_event_index()
6225 	rb = rcu_dereference(event->rb);  in perf_event_init_userpage()
6229 	userpg = rb->user_page;  in perf_event_init_userpage()
6232 	userpg->cap_bit0_is_deprecated = 1;  in perf_event_init_userpage()
6233 	userpg->size = offsetof(struct perf_event_mmap_page, __reserved);  in perf_event_init_userpage()
6234 	userpg->data_offset = PAGE_SIZE;  in perf_event_init_userpage()
6235 	userpg->data_size = perf_data_size(rb);  in perf_event_init_userpage()
6258 	rb = rcu_dereference(event->rb);  in perf_event_update_userpage()
6273 	userpg = rb->user_page;  in perf_event_update_userpage()
6279 	++userpg->lock;  in perf_event_update_userpage()
6281 	userpg->index = perf_event_index(event);  in perf_event_update_userpage()
6282 	userpg->offset = perf_event_count(event, false);  in perf_event_update_userpage()
6283 	if (userpg->index)  in perf_event_update_userpage()
6284 		userpg->offset -= local64_read(&event->hw.prev_count);  in perf_event_update_userpage()
6286 	userpg->time_enabled = enabled +  in perf_event_update_userpage()
6287 			atomic64_read(&event->child_total_time_enabled);  in perf_event_update_userpage()
6289 	userpg->time_running = running +  in perf_event_update_userpage()
6290 			atomic64_read(&event->child_total_time_running);  in perf_event_update_userpage()
6295 	++userpg->lock;  in perf_event_update_userpage()
6304 	struct perf_event *event = vmf->vma->vm_file->private_data;  in perf_mmap_fault()
6308 	if (vmf->flags & FAULT_FLAG_MKWRITE) {  in perf_mmap_fault()
6309 		if (vmf->pgoff == 0)  in perf_mmap_fault()
6315 	rb = rcu_dereference(event->rb);  in perf_mmap_fault()
6319 	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))  in perf_mmap_fault()
6322 	vmf->page = perf_mmap_to_page(rb, vmf->pgoff);  in perf_mmap_fault()
6323 	if (!vmf->page)  in perf_mmap_fault()
6326 	get_page(vmf->page);  in perf_mmap_fault()
6327 	vmf->page->mapping = vmf->vma->vm_file->f_mapping;  in perf_mmap_fault()
6328 	vmf->page->index   = vmf->pgoff;  in perf_mmap_fault()
6343 	WARN_ON_ONCE(event->parent);  in ring_buffer_attach()
6345 	if (event->rb) {  in ring_buffer_attach()
6348 		 * event->rb_entry and wait/clear when adding event->rb_entry.  in ring_buffer_attach()
6350 		WARN_ON_ONCE(event->rcu_pending);  in ring_buffer_attach()
6352 		old_rb = event->rb;  in ring_buffer_attach()
6353 		spin_lock_irqsave(&old_rb->event_lock, flags);  in ring_buffer_attach()
6354 		list_del_rcu(&event->rb_entry);  in ring_buffer_attach()
6355 		spin_unlock_irqrestore(&old_rb->event_lock, flags);  in ring_buffer_attach()
6357 		event->rcu_batches = get_state_synchronize_rcu();  in ring_buffer_attach()
6358 		event->rcu_pending = 1;  in ring_buffer_attach()
6362 		if (event->rcu_pending) {  in ring_buffer_attach()
6363 			cond_synchronize_rcu(event->rcu_batches);  in ring_buffer_attach()
6364 			event->rcu_pending = 0;  in ring_buffer_attach()
6367 		spin_lock_irqsave(&rb->event_lock, flags);  in ring_buffer_attach()
6368 		list_add_rcu(&event->rb_entry, &rb->event_list);  in ring_buffer_attach()
6369 		spin_unlock_irqrestore(&rb->event_lock, flags);  in ring_buffer_attach()
6379 	 * mid-air, but then again, whoever does it like this is  in ring_buffer_attach()
6385 	rcu_assign_pointer(event->rb, rb);  in ring_buffer_attach()
6394 		wake_up_all(&event->waitq);  in ring_buffer_attach()
6402 	if (event->parent)  in ring_buffer_wakeup()
6403 		event = event->parent;  in ring_buffer_wakeup()
6406 	rb = rcu_dereference(event->rb);  in ring_buffer_wakeup()
6408 		list_for_each_entry_rcu(event, &rb->event_list, rb_entry)  in ring_buffer_wakeup()
6409 			wake_up_all(&event->waitq);  in ring_buffer_wakeup()
6418 	if (event->parent)  in ring_buffer_get()
6419 		event = event->parent;  in ring_buffer_get()
6422 	rb = rcu_dereference(event->rb);  in ring_buffer_get()
6424 		if (!refcount_inc_not_zero(&rb->refcount))  in ring_buffer_get()
6434 	if (!refcount_dec_and_test(&rb->refcount))  in ring_buffer_put()
6437 	WARN_ON_ONCE(!list_empty(&rb->event_list));  in ring_buffer_put()
6439 	call_rcu(&rb->rcu_head, rb_free_rcu);  in ring_buffer_put()
6444 	struct perf_event *event = vma->vm_file->private_data;  in perf_mmap_open()
6446 	atomic_inc(&event->mmap_count);  in perf_mmap_open()
6447 	atomic_inc(&event->rb->mmap_count);  in perf_mmap_open()
6449 	if (vma->vm_pgoff)  in perf_mmap_open()
6450 		atomic_inc(&event->rb->aux_mmap_count);  in perf_mmap_open()
6452 	if (event->pmu->event_mapped)  in perf_mmap_open()
6453 		event->pmu->event_mapped(event, vma->vm_mm);  in perf_mmap_open()
6468 	struct perf_event *event = vma->vm_file->private_data;  in perf_mmap_close()
6470 	struct user_struct *mmap_user = rb->mmap_user;  in perf_mmap_close()
6471 	int mmap_locked = rb->mmap_locked;  in perf_mmap_close()
6475 	if (event->pmu->event_unmapped)  in perf_mmap_close()
6476 		event->pmu->event_unmapped(event, vma->vm_mm);  in perf_mmap_close()
6479 	 * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex  in perf_mmap_close()
6482 	if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&  in perf_mmap_close()
6483 	    atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) {  in perf_mmap_close()
6493 		atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->locked_vm);  in perf_mmap_close()
6494 		atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm);  in perf_mmap_close()
6498 		WARN_ON_ONCE(refcount_read(&rb->aux_refcount));  in perf_mmap_close()
6500 		mutex_unlock(&rb->aux_mutex);  in perf_mmap_close()
6503 	if (atomic_dec_and_test(&rb->mmap_count))  in perf_mmap_close()
6506 	if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))  in perf_mmap_close()
6510 	mutex_unlock(&event->mmap_mutex);  in perf_mmap_close()
6523 	list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {  in perf_mmap_close()
6524 		if (!atomic_long_inc_not_zero(&event->refcount)) {  in perf_mmap_close()
6526 			 * This event is en-route to free_event() which will  in perf_mmap_close()
6533 		mutex_lock(&event->mmap_mutex);  in perf_mmap_close()
6544 		if (event->rb == rb)  in perf_mmap_close()
6547 		mutex_unlock(&event->mmap_mutex);  in perf_mmap_close()
6559 	 * It could be there's still a few 0-ref events on the list; they'll  in perf_mmap_close()
6560 	 * get cleaned up by free_event() -- they'll also still have their  in perf_mmap_close()
6567 	atomic_long_sub((size >> PAGE_SHIFT) + 1 - mmap_locked,  in perf_mmap_close()
6568 			&mmap_user->locked_vm);  in perf_mmap_close()
6569 	atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm);  in perf_mmap_close()
6585 	struct perf_event *event = file->private_data;  in perf_mmap()
6597 	 * Don't allow mmap() of inherited per-task counters. This would  in perf_mmap()
6601 	if (event->cpu == -1 && event->attr.inherit)  in perf_mmap()
6602 		return -EINVAL;  in perf_mmap()
6604 	if (!(vma->vm_flags & VM_SHARED))  in perf_mmap()
6605 		return -EINVAL;  in perf_mmap()
6611 	vma_size = vma->vm_end - vma->vm_start;  in perf_mmap()
6613 	if (vma->vm_pgoff == 0) {  in perf_mmap()
6614 		nr_pages = (vma_size / PAGE_SIZE) - 1;  in perf_mmap()
6617 		 * AUX area mapping: if rb->aux_nr_pages != 0, it's already  in perf_mmap()
6623 		if (!event->rb)  in perf_mmap()
6624 			return -EINVAL;  in perf_mmap()
6628 			return -ENOMEM;  in perf_mmap()
6630 		mutex_lock(&event->mmap_mutex);  in perf_mmap()
6631 		ret = -EINVAL;  in perf_mmap()
6633 		rb = event->rb;  in perf_mmap()
6637 		aux_mutex = &rb->aux_mutex;  in perf_mmap()
6640 		aux_offset = READ_ONCE(rb->user_page->aux_offset);  in perf_mmap()
6641 		aux_size = READ_ONCE(rb->user_page->aux_size);  in perf_mmap()
6646 		if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)  in perf_mmap()
6650 		if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)  in perf_mmap()
6657 		if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)  in perf_mmap()
6663 		if (!atomic_inc_not_zero(&rb->mmap_count))  in perf_mmap()
6667 			atomic_inc(&rb->aux_mmap_count);  in perf_mmap()
6672 		atomic_set(&rb->aux_mmap_count, 1);  in perf_mmap()
6679 	 * If we have rb pages ensure they're a power-of-two number, so we  in perf_mmap()
6683 		return -EINVAL;  in perf_mmap()
6686 		return -EINVAL;  in perf_mmap()
6688 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_mmap()
6690 	mutex_lock(&event->mmap_mutex);  in perf_mmap()
6691 	if (event->rb) {  in perf_mmap()
6692 		if (data_page_nr(event->rb) != nr_pages) {  in perf_mmap()
6693 			ret = -EINVAL;  in perf_mmap()
6697 		if (!atomic_inc_not_zero(&event->rb->mmap_count)) {  in perf_mmap()
6703 			mutex_unlock(&event->mmap_mutex);  in perf_mmap()
6713 	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);  in perf_mmap()
6720 	user_locked = atomic_long_read(&user->locked_vm);  in perf_mmap()
6724 	 *     user->locked_vm > user_lock_limit  in perf_mmap()
6735 		extra = user_locked - user_lock_limit;  in perf_mmap()
6736 		user_extra -= extra;  in perf_mmap()
6741 	locked = atomic64_read(&vma->vm_mm->pinned_vm) + extra;  in perf_mmap()
6745 		ret = -EPERM;  in perf_mmap()
6749 	WARN_ON(!rb && event->rb);  in perf_mmap()
6751 	if (vma->vm_flags & VM_WRITE)  in perf_mmap()
6756 			      event->attr.watermark ? event->attr.wakeup_watermark : 0,  in perf_mmap()
6757 			      event->cpu, flags);  in perf_mmap()
6760 			ret = -ENOMEM;  in perf_mmap()
6764 		atomic_set(&rb->mmap_count, 1);  in perf_mmap()
6765 		rb->mmap_user = get_current_user();  in perf_mmap()
6766 		rb->mmap_locked = extra;  in perf_mmap()
6774 		ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,  in perf_mmap()
6775 				   event->attr.aux_watermark, flags);  in perf_mmap()
6777 			rb->aux_mmap_locked = extra;  in perf_mmap()
6782 		atomic_long_add(user_extra, &user->locked_vm);  in perf_mmap()
6783 		atomic64_add(extra, &vma->vm_mm->pinned_vm);  in perf_mmap()
6785 		atomic_inc(&event->mmap_count);  in perf_mmap()
6787 		atomic_dec(&rb->mmap_count);  in perf_mmap()
6792 	mutex_unlock(&event->mmap_mutex);  in perf_mmap()
6799 	vma->vm_ops = &perf_mmap_vmops;  in perf_mmap()
6801 	if (event->pmu->event_mapped)  in perf_mmap()
6802 		event->pmu->event_mapped(event, vma->vm_mm);  in perf_mmap()
6810 	struct perf_event *event = filp->private_data;  in perf_fasync()
6814 	retval = fasync_helper(fd, filp, on, &event->fasync);  in perf_fasync()
6837  * to user-space before waking everybody up.
6844 	if (event->pending_kill) {  in perf_event_wakeup()
6845 		kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);  in perf_event_wakeup()
6846 		event->pending_kill = 0;  in perf_event_wakeup()
6854 	 * ctx->task or current has changed in the meantime. This can be the  in perf_sigtrap()
6857 	if (WARN_ON_ONCE(event->ctx->task != current))  in perf_sigtrap()
6864 	if (current->flags & PF_EXITING)  in perf_sigtrap()
6867 	send_sig_perf((void __user *)event->pending_addr,  in perf_sigtrap()
6868 		      event->orig_type, event->attr.sig_data);  in perf_sigtrap()
6872  * Deliver the pending work in-event-context or follow the context.
6876 	int cpu = READ_ONCE(event->oncpu);  in __perf_pending_disable()
6889 		if (event->pending_disable) {  in __perf_pending_disable()
6890 			event->pending_disable = 0;  in __perf_pending_disable()
6897 	 *  CPU-A			CPU-B  in __perf_pending_disable()
6900 	 *    @pending_disable = CPU-A;  in __perf_pending_disable()
6903 	 *  sched-out  in __perf_pending_disable()
6904 	 *    @pending_disable = -1;  in __perf_pending_disable()
6906 	 *				sched-in  in __perf_pending_disable()
6908 	 *				  @pending_disable = CPU-B;  in __perf_pending_disable()
6914 	 * But the event runs on CPU-B and wants disabling there.  in __perf_pending_disable()
6916 	irq_work_queue_on(&event->pending_disable_irq, cpu);  in __perf_pending_disable()
6946 	 * The wakeup isn't bound to the context of the event -- it can happen  in perf_pending_irq()
6949 	if (event->pending_wakeup) {  in perf_pending_irq()
6950 		event->pending_wakeup = 0;  in perf_pending_irq()
6964 	 * All accesses to the event must belong to the same implicit RCU read-side  in perf_pending_task()
6965 	 * critical section as the ->pending_work reset. See comment in  in perf_pending_task()
6975 	if (event->pending_work) {  in perf_pending_task()
6976 		event->pending_work = 0;  in perf_pending_task()
6978 		local_dec(&event->ctx->nr_no_switch_fast);  in perf_pending_task()
6979 		rcuwait_wake_up(&event->pending_work_wait);  in perf_pending_task()
6990 DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
6991 DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
6992 DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
7000 	static_call_update(__perf_guest_state, cbs->state);  in perf_register_guest_info_callbacks()
7001 	static_call_update(__perf_guest_get_ip, cbs->get_ip);  in perf_register_guest_info_callbacks()
7003 	/* Implementing ->handle_intel_pt_intr is optional. */  in perf_register_guest_info_callbacks()
7004 	if (cbs->handle_intel_pt_intr)  in perf_register_guest_info_callbacks()
7006 				   cbs->handle_intel_pt_intr);  in perf_register_guest_info_callbacks()
7045 		regs_user->abi = perf_reg_abi(current);  in perf_sample_regs_user()
7046 		regs_user->regs = regs;  in perf_sample_regs_user()
7047 	} else if (!(current->flags & PF_KTHREAD)) {  in perf_sample_regs_user()
7050 		regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;  in perf_sample_regs_user()
7051 		regs_user->regs = NULL;  in perf_sample_regs_user()
7058 	regs_intr->regs = regs;  in perf_sample_regs_intr()
7059 	regs_intr->abi  = perf_reg_abi(current);  in perf_sample_regs_intr()
7077 	return TASK_SIZE - addr;  in perf_ustack_task_size()
7092 	 * - TASK_SIZE  in perf_sample_ustack_size()
7095 	 * - remaining sample size  in perf_sample_ustack_size()
7112 		stack_size = USHRT_MAX - header_size - sizeof(u64);  in perf_sample_ustack_size()
7135 		 *   - the size requested by user or the best one we can fit  in perf_output_sample_ustack()
7138 		 *   - user stack dump data  in perf_output_sample_ustack()
7140 		 *   - the actual dumped size  in perf_output_sample_ustack()
7149 		dyn_size = dump_size - rem;  in perf_output_sample_ustack()
7162 	struct perf_event *sampler = event->aux_event;  in perf_prepare_sample_aux()
7165 	data->aux_size = 0;  in perf_prepare_sample_aux()
7170 	if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE))  in perf_prepare_sample_aux()
7173 	if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id()))  in perf_prepare_sample_aux()
7184 	if (READ_ONCE(rb->aux_in_sampling)) {  in perf_prepare_sample_aux()
7185 		data->aux_size = 0;  in perf_prepare_sample_aux()
7188 		data->aux_size = ALIGN(size, sizeof(u64));  in perf_prepare_sample_aux()
7193 	return data->aux_size;  in perf_prepare_sample_aux()
7205 	 * Normal ->start()/->stop() callbacks run in IRQ mode in scheduler  in perf_pmu_snapshot_aux()
7207 	 * the IRQ ones, that is, for example, re-starting an event that's just  in perf_pmu_snapshot_aux()
7218 	WRITE_ONCE(rb->aux_in_sampling, 1);  in perf_pmu_snapshot_aux()
7221 	ret = event->pmu->snapshot_aux(event, handle, size);  in perf_pmu_snapshot_aux()
7224 	WRITE_ONCE(rb->aux_in_sampling, 0);  in perf_pmu_snapshot_aux()
7234 	struct perf_event *sampler = event->aux_event;  in perf_aux_sample_output()
7239 	if (WARN_ON_ONCE(!sampler || !data->aux_size))  in perf_aux_sample_output()
7246 	size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size);  in perf_aux_sample_output()
7250 	 * non-zero surplus that it didn't copy), which in its current  in perf_aux_sample_output()
7258 	 * The pad comes from ALIGN()ing data->aux_size up to u64 in  in perf_aux_sample_output()
7261 	pad = data->aux_size - size;  in perf_aux_sample_output()
7275  * A set of common sample data types saved even for non-sample records
7276  * when event->attr.sample_id_all is set.
7286 	data->type = event->attr.sample_type;  in __perf_event_header__init_id()
7287 	data->sample_flags |= data->type & PERF_SAMPLE_ID_ALL;  in __perf_event_header__init_id()
7291 		data->tid_entry.pid = perf_event_pid(event, current);  in __perf_event_header__init_id()
7292 		data->tid_entry.tid = perf_event_tid(event, current);  in __perf_event_header__init_id()
7296 		data->time = perf_event_clock(event);  in __perf_event_header__init_id()
7299 		data->id = primary_event_id(event);  in __perf_event_header__init_id()
7302 		data->stream_id = event->id;  in __perf_event_header__init_id()
7305 		data->cpu_entry.cpu	 = raw_smp_processor_id();  in __perf_event_header__init_id()
7306 		data->cpu_entry.reserved = 0;  in __perf_event_header__init_id()
7314 	if (event->attr.sample_id_all) {  in perf_event_header__init_id()
7315 		header->size += event->id_header_size;  in perf_event_header__init_id()
7316 		__perf_event_header__init_id(data, event, event->attr.sample_type);  in perf_event_header__init_id()
7323 	u64 sample_type = data->type;  in __perf_event__output_id_sample()
7326 		perf_output_put(handle, data->tid_entry);  in __perf_event__output_id_sample()
7329 		perf_output_put(handle, data->time);  in __perf_event__output_id_sample()
7332 		perf_output_put(handle, data->id);  in __perf_event__output_id_sample()
7335 		perf_output_put(handle, data->stream_id);  in __perf_event__output_id_sample()
7338 		perf_output_put(handle, data->cpu_entry);  in __perf_event__output_id_sample()
7341 		perf_output_put(handle, data->id);  in __perf_event__output_id_sample()
7348 	if (event->attr.sample_id_all)  in perf_event__output_id_sample()
7356 	u64 read_format = event->attr.read_format;  in perf_output_read_one()
7360 	values[n++] = perf_event_count(event, has_inherit_and_sample_read(&event->attr));  in perf_output_read_one()
7363 			atomic64_read(&event->child_total_time_enabled);  in perf_output_read_one()
7367 			atomic64_read(&event->child_total_time_running);  in perf_output_read_one()
7372 		values[n++] = atomic64_read(&event->lost_samples);  in perf_output_read_one()
7381 	struct perf_event *leader = event->group_leader, *sub;  in perf_output_read_group()
7382 	u64 read_format = event->attr.read_format;  in perf_output_read_group()
7386 	bool self = has_inherit_and_sample_read(&event->attr);  in perf_output_read_group()
7394 	values[n++] = 1 + leader->nr_siblings;  in perf_output_read_group()
7403 	    (leader->state == PERF_EVENT_STATE_ACTIVE))  in perf_output_read_group()
7404 		leader->pmu->read(leader);  in perf_output_read_group()
7410 		values[n++] = atomic64_read(&leader->lost_samples);  in perf_output_read_group()
7418 		    (sub->state == PERF_EVENT_STATE_ACTIVE))  in perf_output_read_group()
7419 			sub->pmu->read(sub);  in perf_output_read_group()
7425 			values[n++] = atomic64_read(&sub->lost_samples);  in perf_output_read_group()
7440  * child list, not to mention that its impossible to IPI the children running
7443  * Instead the combination of PERF_SAMPLE_READ and inherit will track per-thread
7451 	u64 read_format = event->attr.read_format;  in perf_output_read()
7465 	if (event->attr.read_format & PERF_FORMAT_GROUP)  in perf_output_read()
7476 	u64 sample_type = data->type;  in perf_output_sample()
7481 		perf_output_put(handle, data->id);  in perf_output_sample()
7484 		perf_output_put(handle, data->ip);  in perf_output_sample()
7487 		perf_output_put(handle, data->tid_entry);  in perf_output_sample()
7490 		perf_output_put(handle, data->time);  in perf_output_sample()
7493 		perf_output_put(handle, data->addr);  in perf_output_sample()
7496 		perf_output_put(handle, data->id);  in perf_output_sample()
7499 		perf_output_put(handle, data->stream_id);  in perf_output_sample()
7502 		perf_output_put(handle, data->cpu_entry);  in perf_output_sample()
7505 		perf_output_put(handle, data->period);  in perf_output_sample()
7513 		size += data->callchain->nr;  in perf_output_sample()
7515 		__output_copy(handle, data->callchain, size);  in perf_output_sample()
7519 		struct perf_raw_record *raw = data->raw;  in perf_output_sample()
7522 			struct perf_raw_frag *frag = &raw->frag;  in perf_output_sample()
7524 			perf_output_put(handle, raw->size);  in perf_output_sample()
7526 				if (frag->copy) {  in perf_output_sample()
7527 					__output_custom(handle, frag->copy,  in perf_output_sample()
7528 							frag->data, frag->size);  in perf_output_sample()
7530 					__output_copy(handle, frag->data,  in perf_output_sample()
7531 						      frag->size);  in perf_output_sample()
7535 				frag = frag->next;  in perf_output_sample()
7537 			if (frag->pad)  in perf_output_sample()
7538 				__output_skip(handle, NULL, frag->pad);  in perf_output_sample()
7552 		if (data->br_stack) {  in perf_output_sample()
7555 			size = data->br_stack->nr  in perf_output_sample()
7558 			perf_output_put(handle, data->br_stack->nr);  in perf_output_sample()
7560 				perf_output_put(handle, data->br_stack->hw_idx);  in perf_output_sample()
7561 			perf_output_copy(handle, data->br_stack->entries, size);  in perf_output_sample()
7566 			if (data->br_stack_cntr) {  in perf_output_sample()
7567 				size = data->br_stack->nr * sizeof(u64);  in perf_output_sample()
7568 				perf_output_copy(handle, data->br_stack_cntr, size);  in perf_output_sample()
7580 		u64 abi = data->regs_user.abi;  in perf_output_sample()
7589 			u64 mask = event->attr.sample_regs_user;  in perf_output_sample()
7591 						data->regs_user.regs,  in perf_output_sample()
7598 					  data->stack_user_size,  in perf_output_sample()
7599 					  data->regs_user.regs);  in perf_output_sample()
7603 		perf_output_put(handle, data->weight.full);  in perf_output_sample()
7606 		perf_output_put(handle, data->data_src.val);  in perf_output_sample()
7609 		perf_output_put(handle, data->txn);  in perf_output_sample()
7612 		u64 abi = data->regs_intr.abi;  in perf_output_sample()
7620 			u64 mask = event->attr.sample_regs_intr;  in perf_output_sample()
7623 						data->regs_intr.regs,  in perf_output_sample()
7629 		perf_output_put(handle, data->phys_addr);  in perf_output_sample()
7632 		perf_output_put(handle, data->cgroup);  in perf_output_sample()
7635 		perf_output_put(handle, data->data_page_size);  in perf_output_sample()
7638 		perf_output_put(handle, data->code_page_size);  in perf_output_sample()
7641 		perf_output_put(handle, data->aux_size);  in perf_output_sample()
7643 		if (data->aux_size)  in perf_output_sample()
7647 	if (!event->attr.watermark) {  in perf_output_sample()
7648 		int wakeup_events = event->attr.wakeup_events;  in perf_output_sample()
7651 			struct perf_buffer *rb = handle->rb;  in perf_output_sample()
7652 			int events = local_inc_return(&rb->events);  in perf_output_sample()
7655 				local_sub(wakeup_events, &rb->events);  in perf_output_sample()
7656 				local_inc(&rb->wakeup);  in perf_output_sample()
7679 		 * Try IRQ-safe get_user_page_fast_only first.  in perf_virt_to_phys()
7682 		if (current->mm != NULL) {  in perf_virt_to_phys()
7767 	 * Software page-table walkers must disable IRQs,  in perf_get_page_size()
7772 	mm = current->mm;  in perf_get_page_size()
7793 	bool kernel = !event->attr.exclude_callchain_kernel;  in perf_callchain()
7794 	bool user   = !event->attr.exclude_callchain_user;  in perf_callchain()
7795 	/* Disallow cross-task user callchains. */  in perf_callchain()
7796 	bool crosstask = event->ctx->task && event->ctx->task != current;  in perf_callchain()
7797 	const u32 max_stack = event->attr.sample_max_stack;  in perf_callchain()
7817 	u64 sample_type = event->attr.sample_type;  in perf_prepare_sample()
7831 	filtered_sample_type &= ~data->sample_flags;  in perf_prepare_sample()
7834 		/* Make sure it has the correct data->type for output */  in perf_prepare_sample()
7835 		data->type = event->attr.sample_type;  in perf_prepare_sample()
7842 		data->ip = perf_instruction_pointer(regs);  in perf_prepare_sample()
7843 		data->sample_flags |= PERF_SAMPLE_IP;  in perf_prepare_sample()
7850 		data->raw = NULL;  in perf_prepare_sample()
7851 		data->dyn_size += sizeof(u64);  in perf_prepare_sample()
7852 		data->sample_flags |= PERF_SAMPLE_RAW;  in perf_prepare_sample()
7856 		data->br_stack = NULL;  in perf_prepare_sample()
7857 		data->dyn_size += sizeof(u64);  in perf_prepare_sample()
7858 		data->sample_flags |= PERF_SAMPLE_BRANCH_STACK;  in perf_prepare_sample()
7862 		perf_sample_regs_user(&data->regs_user, regs);  in perf_prepare_sample()
7869 	if ((sample_type & ~data->sample_flags) & PERF_SAMPLE_REGS_USER) {  in perf_prepare_sample()
7873 		if (data->regs_user.regs) {  in perf_prepare_sample()
7874 			u64 mask = event->attr.sample_regs_user;  in perf_prepare_sample()
7878 		data->dyn_size += size;  in perf_prepare_sample()
7879 		data->sample_flags |= PERF_SAMPLE_REGS_USER;  in perf_prepare_sample()
7889 		u16 stack_size = event->attr.sample_stack_user;  in perf_prepare_sample()
7894 						     data->regs_user.regs);  in perf_prepare_sample()
7904 		data->stack_user_size = stack_size;  in perf_prepare_sample()
7905 		data->dyn_size += size;  in perf_prepare_sample()
7906 		data->sample_flags |= PERF_SAMPLE_STACK_USER;  in perf_prepare_sample()
7910 		data->weight.full = 0;  in perf_prepare_sample()
7911 		data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE;  in perf_prepare_sample()
7915 		data->data_src.val = PERF_MEM_NA;  in perf_prepare_sample()
7916 		data->sample_flags |= PERF_SAMPLE_DATA_SRC;  in perf_prepare_sample()
7920 		data->txn = 0;  in perf_prepare_sample()
7921 		data->sample_flags |= PERF_SAMPLE_TRANSACTION;  in perf_prepare_sample()
7925 		data->addr = 0;  in perf_prepare_sample()
7926 		data->sample_flags |= PERF_SAMPLE_ADDR;  in perf_prepare_sample()
7933 		perf_sample_regs_intr(&data->regs_intr, regs);  in perf_prepare_sample()
7935 		if (data->regs_intr.regs) {  in perf_prepare_sample()
7936 			u64 mask = event->attr.sample_regs_intr;  in perf_prepare_sample()
7941 		data->dyn_size += size;  in perf_prepare_sample()
7942 		data->sample_flags |= PERF_SAMPLE_REGS_INTR;  in perf_prepare_sample()
7946 		data->phys_addr = perf_virt_to_phys(data->addr);  in perf_prepare_sample()
7947 		data->sample_flags |= PERF_SAMPLE_PHYS_ADDR;  in perf_prepare_sample()
7955 		cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup;  in perf_prepare_sample()
7956 		data->cgroup = cgroup_id(cgrp);  in perf_prepare_sample()
7957 		data->sample_flags |= PERF_SAMPLE_CGROUP;  in perf_prepare_sample()
7963 	 * require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr,  in perf_prepare_sample()
7967 		data->data_page_size = perf_get_page_size(data->addr);  in perf_prepare_sample()
7968 		data->sample_flags |= PERF_SAMPLE_DATA_PAGE_SIZE;  in perf_prepare_sample()
7972 		data->code_page_size = perf_get_page_size(data->ip);  in perf_prepare_sample()
7973 		data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE;  in perf_prepare_sample()
7988 		size = min_t(size_t, U16_MAX - header_size,  in perf_prepare_sample()
7989 			     event->attr.aux_sample_size);  in perf_prepare_sample()
7994 		data->dyn_size += size + sizeof(u64); /* size above */  in perf_prepare_sample()
7995 		data->sample_flags |= PERF_SAMPLE_AUX;  in perf_prepare_sample()
8004 	header->type = PERF_RECORD_SAMPLE;  in perf_prepare_header()
8005 	header->size = perf_sample_data_size(data, event);  in perf_prepare_header()
8006 	header->misc = perf_misc_flags(regs);  in perf_prepare_header()
8016 	WARN_ON_ONCE(header->size & 7);  in perf_prepare_header()
8096 			.size = sizeof(read_event) + event->read_size,  in perf_event_read_event()
8124 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {  in perf_iterate_ctx()
8126 			if (event->state < PERF_EVENT_STATE_INACTIVE)  in perf_iterate_ctx()
8141 	list_for_each_entry_rcu(event, &pel->list, sb_list) {  in perf_iterate_sb_cpu()
8144 		 * if we observe event->ctx, both event and ctx will be  in perf_iterate_sb_cpu()
8147 		if (!smp_load_acquire(&event->ctx))  in perf_iterate_sb_cpu()
8150 		if (event->state < PERF_EVENT_STATE_INACTIVE)  in perf_iterate_sb_cpu()
8159  * Iterate all events that need to receive side-band events.
8185 	ctx = rcu_dereference(current->perf_event_ctxp);  in perf_iterate_sb()
8194  * Clear all file-based filters at exec, they'll have to be
8195  * re-instated when/if these objects are mmapped again.
8207 	raw_spin_lock_irqsave(&ifh->lock, flags);  in perf_event_addr_filters_exec()
8208 	list_for_each_entry(filter, &ifh->list, entry) {  in perf_event_addr_filters_exec()
8209 		if (filter->path.dentry) {  in perf_event_addr_filters_exec()
8210 			event->addr_filter_ranges[count].start = 0;  in perf_event_addr_filters_exec()
8211 			event->addr_filter_ranges[count].size = 0;  in perf_event_addr_filters_exec()
8219 		event->addr_filters_gen++;  in perf_event_addr_filters_exec()
8220 	raw_spin_unlock_irqrestore(&ifh->lock, flags);  in perf_event_addr_filters_exec()
8249 	struct perf_event *parent = event->parent;  in __perf_event_output_stop()
8251 	struct perf_buffer *rb = ro->rb;  in __perf_event_output_stop()
8264 	 * ring-buffer, but it will be the child that's actually using it.  in __perf_event_output_stop()
8269 	 * So ring_buffer_attach() has to stop an aux event before re-assigning  in __perf_event_output_stop()
8272 	if (rcu_dereference(parent->rb) == rb)  in __perf_event_output_stop()
8273 		ro->err = __perf_event_stop(&sd);  in __perf_event_output_stop()
8281 		.rb	= event->rb,  in __perf_pmu_output_stop()
8285 	perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false);  in __perf_pmu_output_stop()
8286 	if (cpuctx->task_ctx)  in __perf_pmu_output_stop()
8287 		perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop,  in __perf_pmu_output_stop()
8301 	list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) {  in perf_pmu_output_stop()
8303 		 * For per-CPU events, we need to make sure that neither they  in perf_pmu_output_stop()
8304 		 * nor their children are running; for cpu==-1 events it's  in perf_pmu_output_stop()
8308 		cpu = iter->cpu;  in perf_pmu_output_stop()
8309 		if (cpu == -1)  in perf_pmu_output_stop()
8310 			cpu = READ_ONCE(iter->oncpu);  in perf_pmu_output_stop()
8312 		if (cpu == -1)  in perf_pmu_output_stop()
8316 		if (err == -EAGAIN) {  in perf_pmu_output_stop()
8325  * task tracking -- fork/exit
8347 	return event->attr.comm  || event->attr.mmap ||  in perf_event_task_match()
8348 	       event->attr.mmap2 || event->attr.mmap_data ||  in perf_event_task_match()
8349 	       event->attr.task;  in perf_event_task_match()
8358 	struct task_struct *task = task_event->task;  in perf_event_task_output()
8359 	int ret, size = task_event->event_id.header.size;  in perf_event_task_output()
8364 	perf_event_header__init_id(&task_event->event_id.header, &sample, event);  in perf_event_task_output()
8367 				task_event->event_id.header.size);  in perf_event_task_output()
8371 	task_event->event_id.pid = perf_event_pid(event, task);  in perf_event_task_output()
8372 	task_event->event_id.tid = perf_event_tid(event, task);  in perf_event_task_output()
8374 	if (task_event->event_id.header.type == PERF_RECORD_EXIT) {  in perf_event_task_output()
8375 		task_event->event_id.ppid = perf_event_pid(event,  in perf_event_task_output()
8376 							task->real_parent);  in perf_event_task_output()
8377 		task_event->event_id.ptid = perf_event_pid(event,  in perf_event_task_output()
8378 							task->real_parent);  in perf_event_task_output()
8380 		task_event->event_id.ppid = perf_event_pid(event, current);  in perf_event_task_output()
8381 		task_event->event_id.ptid = perf_event_tid(event, current);  in perf_event_task_output()
8384 	task_event->event_id.time = perf_event_clock(event);  in perf_event_task_output()
8386 	perf_output_put(&handle, task_event->event_id);  in perf_event_task_output()
8392 	task_event->event_id.header.size = size;  in perf_event_task_output()
8453 	return event->attr.comm;  in perf_event_comm_match()
8462 	int size = comm_event->event_id.header.size;  in perf_event_comm_output()
8468 	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);  in perf_event_comm_output()
8470 				comm_event->event_id.header.size);  in perf_event_comm_output()
8475 	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);  in perf_event_comm_output()
8476 	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);  in perf_event_comm_output()
8478 	perf_output_put(&handle, comm_event->event_id);  in perf_event_comm_output()
8479 	__output_copy(&handle, comm_event->comm,  in perf_event_comm_output()
8480 				   comm_event->comm_size);  in perf_event_comm_output()
8486 	comm_event->event_id.header.size = size;  in perf_event_comm_output()
8495 	strscpy(comm, comm_event->task->comm, sizeof(comm));  in perf_event_comm_event()
8498 	comm_event->comm = comm;  in perf_event_comm_event()
8499 	comm_event->comm_size = size;  in perf_event_comm_event()
8501 	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;  in perf_event_comm_event()
8552 	return event->attr.namespaces;  in perf_event_namespaces_match()
8561 	u16 header_size = namespaces_event->event_id.header.size;  in perf_event_namespaces_output()
8567 	perf_event_header__init_id(&namespaces_event->event_id.header,  in perf_event_namespaces_output()
8570 				namespaces_event->event_id.header.size);  in perf_event_namespaces_output()
8574 	namespaces_event->event_id.pid = perf_event_pid(event,  in perf_event_namespaces_output()
8575 							namespaces_event->task);  in perf_event_namespaces_output()
8576 	namespaces_event->event_id.tid = perf_event_tid(event,  in perf_event_namespaces_output()
8577 							namespaces_event->task);  in perf_event_namespaces_output()
8579 	perf_output_put(&handle, namespaces_event->event_id);  in perf_event_namespaces_output()
8585 	namespaces_event->event_id.header.size = header_size;  in perf_event_namespaces_output()
8598 		ns_inode = ns_path.dentry->d_inode;  in perf_fill_ns_link_info()
8599 		ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev);  in perf_fill_ns_link_info()
8600 		ns_link_info->ino = ns_inode->i_ino;  in perf_fill_ns_link_info()
8680 	return event->attr.cgroup;  in perf_event_cgroup_match()
8688 	u16 header_size = cgroup_event->event_id.header.size;  in perf_event_cgroup_output()
8694 	perf_event_header__init_id(&cgroup_event->event_id.header,  in perf_event_cgroup_output()
8697 				cgroup_event->event_id.header.size);  in perf_event_cgroup_output()
8701 	perf_output_put(&handle, cgroup_event->event_id);  in perf_event_cgroup_output()
8702 	__output_copy(&handle, cgroup_event->path, cgroup_event->path_size);  in perf_event_cgroup_output()
8708 	cgroup_event->event_id.header.size = header_size;  in perf_event_cgroup_output()
8737 		cgroup_path(cgrp, pathname, PATH_MAX - sizeof(u64));  in perf_event_cgroup()
8793 	struct vm_area_struct *vma = mmap_event->vma;  in perf_event_mmap_match()
8794 	int executable = vma->vm_flags & VM_EXEC;  in perf_event_mmap_match()
8796 	return (!executable && event->attr.mmap_data) ||  in perf_event_mmap_match()
8797 	       (executable && (event->attr.mmap || event->attr.mmap2));  in perf_event_mmap_match()
8806 	int size = mmap_event->event_id.header.size;  in perf_event_mmap_output()
8807 	u32 type = mmap_event->event_id.header.type;  in perf_event_mmap_output()
8814 	if (event->attr.mmap2) {  in perf_event_mmap_output()
8815 		mmap_event->event_id.header.type = PERF_RECORD_MMAP2;  in perf_event_mmap_output()
8816 		mmap_event->event_id.header.size += sizeof(mmap_event->maj);  in perf_event_mmap_output()
8817 		mmap_event->event_id.header.size += sizeof(mmap_event->min);  in perf_event_mmap_output()
8818 		mmap_event->event_id.header.size += sizeof(mmap_event->ino);  in perf_event_mmap_output()
8819 		mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);  in perf_event_mmap_output()
8820 		mmap_event->event_id.header.size += sizeof(mmap_event->prot);  in perf_event_mmap_output()
8821 		mmap_event->event_id.header.size += sizeof(mmap_event->flags);  in perf_event_mmap_output()
8824 	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);  in perf_event_mmap_output()
8826 				mmap_event->event_id.header.size);  in perf_event_mmap_output()
8830 	mmap_event->event_id.pid = perf_event_pid(event, current);  in perf_event_mmap_output()
8831 	mmap_event->event_id.tid = perf_event_tid(event, current);  in perf_event_mmap_output()
8833 	use_build_id = event->attr.build_id && mmap_event->build_id_size;  in perf_event_mmap_output()
8835 	if (event->attr.mmap2 && use_build_id)  in perf_event_mmap_output()
8836 		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID;  in perf_event_mmap_output()
8838 	perf_output_put(&handle, mmap_event->event_id);  in perf_event_mmap_output()
8840 	if (event->attr.mmap2) {  in perf_event_mmap_output()
8842 			u8 size[4] = { (u8) mmap_event->build_id_size, 0, 0, 0 };  in perf_event_mmap_output()
8845 			__output_copy(&handle, mmap_event->build_id, BUILD_ID_SIZE_MAX);  in perf_event_mmap_output()
8847 			perf_output_put(&handle, mmap_event->maj);  in perf_event_mmap_output()
8848 			perf_output_put(&handle, mmap_event->min);  in perf_event_mmap_output()
8849 			perf_output_put(&handle, mmap_event->ino);  in perf_event_mmap_output()
8850 			perf_output_put(&handle, mmap_event->ino_generation);  in perf_event_mmap_output()
8852 		perf_output_put(&handle, mmap_event->prot);  in perf_event_mmap_output()
8853 		perf_output_put(&handle, mmap_event->flags);  in perf_event_mmap_output()
8856 	__output_copy(&handle, mmap_event->file_name,  in perf_event_mmap_output()
8857 				   mmap_event->file_size);  in perf_event_mmap_output()
8863 	mmap_event->event_id.header.size = size;  in perf_event_mmap_output()
8864 	mmap_event->event_id.header.type = type;  in perf_event_mmap_output()
8869 	struct vm_area_struct *vma = mmap_event->vma;  in perf_event_mmap_event()
8870 	struct file *file = vma->vm_file;  in perf_event_mmap_event()
8879 	if (vma->vm_flags & VM_READ)  in perf_event_mmap_event()
8881 	if (vma->vm_flags & VM_WRITE)  in perf_event_mmap_event()
8883 	if (vma->vm_flags & VM_EXEC)  in perf_event_mmap_event()
8886 	if (vma->vm_flags & VM_MAYSHARE)  in perf_event_mmap_event()
8891 	if (vma->vm_flags & VM_LOCKED)  in perf_event_mmap_event()
8910 		name = file_path(file, buf, PATH_MAX - sizeof(u64));  in perf_event_mmap_event()
8915 		inode = file_inode(vma->vm_file);  in perf_event_mmap_event()
8916 		dev = inode->i_sb->s_dev;  in perf_event_mmap_event()
8917 		ino = inode->i_ino;  in perf_event_mmap_event()
8918 		gen = inode->i_generation;  in perf_event_mmap_event()
8924 		if (vma->vm_ops && vma->vm_ops->name)  in perf_event_mmap_event()
8925 			name = (char *) vma->vm_ops->name(vma);  in perf_event_mmap_event()
8951 	mmap_event->file_name = name;  in perf_event_mmap_event()
8952 	mmap_event->file_size = size;  in perf_event_mmap_event()
8953 	mmap_event->maj = maj;  in perf_event_mmap_event()
8954 	mmap_event->min = min;  in perf_event_mmap_event()
8955 	mmap_event->ino = ino;  in perf_event_mmap_event()
8956 	mmap_event->ino_generation = gen;  in perf_event_mmap_event()
8957 	mmap_event->prot = prot;  in perf_event_mmap_event()
8958 	mmap_event->flags = flags;  in perf_event_mmap_event()
8960 	if (!(vma->vm_flags & VM_EXEC))  in perf_event_mmap_event()
8961 		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;  in perf_event_mmap_event()
8963 	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;  in perf_event_mmap_event()
8966 		build_id_parse_nofault(vma, mmap_event->build_id, &mmap_event->build_id_size);  in perf_event_mmap_event()
8982 	/* d_inode(NULL) won't be equal to any mapped user-space file */  in perf_addr_filter_match()
8983 	if (!filter->path.dentry)  in perf_addr_filter_match()
8986 	if (d_inode(filter->path.dentry) != file_inode(file))  in perf_addr_filter_match()
8989 	if (filter->offset > offset + size)  in perf_addr_filter_match()
8992 	if (filter->offset + filter->size < offset)  in perf_addr_filter_match()
9002 	unsigned long vma_size = vma->vm_end - vma->vm_start;  in perf_addr_filter_vma_adjust()
9003 	unsigned long off = vma->vm_pgoff << PAGE_SHIFT;  in perf_addr_filter_vma_adjust()
9004 	struct file *file = vma->vm_file;  in perf_addr_filter_vma_adjust()
9009 	if (filter->offset < off) {  in perf_addr_filter_vma_adjust()
9010 		fr->start = vma->vm_start;  in perf_addr_filter_vma_adjust()
9011 		fr->size = min(vma_size, filter->size - (off - filter->offset));  in perf_addr_filter_vma_adjust()
9013 		fr->start = vma->vm_start + filter->offset - off;  in perf_addr_filter_vma_adjust()
9014 		fr->size = min(vma->vm_end - fr->start, filter->size);  in perf_addr_filter_vma_adjust()
9031 	if (!vma->vm_file)  in __perf_addr_filters_adjust()
9034 	raw_spin_lock_irqsave(&ifh->lock, flags);  in __perf_addr_filters_adjust()
9035 	list_for_each_entry(filter, &ifh->list, entry) {  in __perf_addr_filters_adjust()
9037 						&event->addr_filter_ranges[count]))  in __perf_addr_filters_adjust()
9044 		event->addr_filters_gen++;  in __perf_addr_filters_adjust()
9045 	raw_spin_unlock_irqrestore(&ifh->lock, flags);  in __perf_addr_filters_adjust()
9062 	if (!(vma->vm_flags & VM_EXEC))  in perf_addr_filters_adjust()
9066 	ctx = rcu_dereference(current->perf_event_ctxp);  in perf_addr_filters_adjust()
9091 			.start  = vma->vm_start,  in perf_event_mmap()
9092 			.len    = vma->vm_end - vma->vm_start,  in perf_event_mmap()
9093 			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,  in perf_event_mmap()
9191 	return event->attr.context_switch;  in perf_event_switch_match()
9204 	/* Only CPU-wide events are allowed to see next/prev pid/tid */  in perf_event_switch_output()
9205 	if (event->ctx->task) {  in perf_event_switch_output()
9206 		se->event_id.header.type = PERF_RECORD_SWITCH;  in perf_event_switch_output()
9207 		se->event_id.header.size = sizeof(se->event_id.header);  in perf_event_switch_output()
9209 		se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE;  in perf_event_switch_output()
9210 		se->event_id.header.size = sizeof(se->event_id);  in perf_event_switch_output()
9211 		se->event_id.next_prev_pid =  in perf_event_switch_output()
9212 					perf_event_pid(event, se->next_prev);  in perf_event_switch_output()
9213 		se->event_id.next_prev_tid =  in perf_event_switch_output()
9214 					perf_event_tid(event, se->next_prev);  in perf_event_switch_output()
9217 	perf_event_header__init_id(&se->event_id.header, &sample, event);  in perf_event_switch_output()
9219 	ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);  in perf_event_switch_output()
9223 	if (event->ctx->task)  in perf_event_switch_output()
9224 		perf_output_put(&handle, se->event_id.header);  in perf_event_switch_output()
9226 		perf_output_put(&handle, se->event_id);  in perf_event_switch_output()
9285 		.stream_id	= event->id,  in perf_log_throttle()
9321 	return event->attr.ksymbol;  in perf_event_ksymbol_match()
9334 	perf_event_header__init_id(&ksymbol_event->event_id.header,  in perf_event_ksymbol_output()
9337 				ksymbol_event->event_id.header.size);  in perf_event_ksymbol_output()
9341 	perf_output_put(&handle, ksymbol_event->event_id);  in perf_event_ksymbol_output()
9342 	__output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len);  in perf_event_ksymbol_output()
9411 	return event->attr.bpf_event;  in perf_event_bpf_match()
9424 	perf_event_header__init_id(&bpf_event->event_id.header,  in perf_event_bpf_output()
9427 				bpf_event->event_id.header.size);  in perf_event_bpf_output()
9431 	perf_output_put(&handle, bpf_event->event_id);  in perf_event_bpf_output()
9444 			   (u64)(unsigned long)prog->bpf_func,  in perf_event_bpf_emit_ksymbols()
9445 			   prog->jited_len, unregister,  in perf_event_bpf_emit_ksymbols()
9446 			   prog->aux->ksym.name);  in perf_event_bpf_emit_ksymbols()
9448 	for (i = 1; i < prog->aux->func_cnt; i++) {  in perf_event_bpf_emit_ksymbols()
9449 		struct bpf_prog *subprog = prog->aux->func[i];  in perf_event_bpf_emit_ksymbols()
9453 			(u64)(unsigned long)subprog->bpf_func,  in perf_event_bpf_emit_ksymbols()
9454 			subprog->jited_len, unregister,  in perf_event_bpf_emit_ksymbols()
9455 			subprog->aux->ksym.name);  in perf_event_bpf_emit_ksymbols()
9487 			.id = prog->aux->id,  in perf_event_bpf_event()
9493 	memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE);  in perf_event_bpf_event()
9513 	return event->attr.text_poke;  in perf_event_text_poke_match()
9527 	perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);  in perf_event_text_poke_output()
9530 				text_poke_event->event_id.header.size);  in perf_event_text_poke_output()
9534 	perf_output_put(&handle, text_poke_event->event_id);  in perf_event_text_poke_output()
9535 	perf_output_put(&handle, text_poke_event->old_len);  in perf_event_text_poke_output()
9536 	perf_output_put(&handle, text_poke_event->new_len);  in perf_event_text_poke_output()
9538 	__output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len);  in perf_event_text_poke_output()
9539 	__output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len);  in perf_event_text_poke_output()
9541 	if (text_poke_event->pad)  in perf_event_text_poke_output()
9542 		__output_copy(&handle, &padding, text_poke_event->pad);  in perf_event_text_poke_output()
9560 	pad  = ALIGN(tot, sizeof(u64)) - tot;  in perf_event_text_poke()
9583 	event->attach_state |= PERF_ATTACH_ITRACE;  in perf_event_itrace_started()
9597 	if (event->parent)  in perf_log_itrace_start()
9598 		event = event->parent;  in perf_log_itrace_start()
9600 	if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) ||  in perf_log_itrace_start()
9601 	    event->attach_state & PERF_ATTACH_ITRACE)  in perf_log_itrace_start()
9632 	if (event->parent)  in perf_report_aux_output_id()
9633 		event = event->parent;  in perf_report_aux_output_id()
9656 	struct hw_perf_event *hwc = &event->hw;  in __perf_event_account_interrupt()
9661 	if (seq != hwc->interrupts_seq) {  in __perf_event_account_interrupt()
9662 		hwc->interrupts_seq = seq;  in __perf_event_account_interrupt()
9663 		hwc->interrupts = 1;  in __perf_event_account_interrupt()
9665 		hwc->interrupts++;  in __perf_event_account_interrupt()
9667 			     hwc->interrupts > max_samples_per_tick)) {  in __perf_event_account_interrupt()
9670 			hwc->interrupts = MAX_INTERRUPTS;  in __perf_event_account_interrupt()
9676 	if (event->attr.freq) {  in __perf_event_account_interrupt()
9678 		s64 delta = now - hwc->freq_time_stamp;  in __perf_event_account_interrupt()
9680 		hwc->freq_time_stamp = now;  in __perf_event_account_interrupt()
9683 			perf_adjust_period(event, delta, hwc->last_period, true);  in __perf_event_account_interrupt()
9701 	if (event->attr.exclude_kernel && !user_mode(regs))  in sample_is_allowed()
9723 	prog = READ_ONCE(event->prog);  in bpf_overflow_handler()
9739 	if (event->overflow_handler_context)  in perf_event_set_bpf_handler()
9741 		return -EINVAL;  in perf_event_set_bpf_handler()
9743 	if (event->prog)  in perf_event_set_bpf_handler()
9744 		return -EEXIST;  in perf_event_set_bpf_handler()
9746 	if (prog->type != BPF_PROG_TYPE_PERF_EVENT)  in perf_event_set_bpf_handler()
9747 		return -EINVAL;  in perf_event_set_bpf_handler()
9749 	if (event->attr.precise_ip &&  in perf_event_set_bpf_handler()
9750 	    prog->call_get_stack &&  in perf_event_set_bpf_handler()
9751 	    (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) ||  in perf_event_set_bpf_handler()
9752 	     event->attr.exclude_callchain_kernel ||  in perf_event_set_bpf_handler()
9753 	     event->attr.exclude_callchain_user)) {  in perf_event_set_bpf_handler()
9763 		return -EPROTO;  in perf_event_set_bpf_handler()
9766 	event->prog = prog;  in perf_event_set_bpf_handler()
9767 	event->bpf_cookie = bpf_cookie;  in perf_event_set_bpf_handler()
9773 	struct bpf_prog *prog = event->prog;  in perf_event_free_bpf_handler()
9778 	event->prog = NULL;  in perf_event_free_bpf_handler()
9793 	return -EOPNOTSUPP;  in perf_event_set_bpf_handler()
9809 	int events = atomic_read(&event->event_limit);  in __perf_event_overflow()
9813 	 * Non-sampling counters might still use the PMI to fold short  in __perf_event_overflow()
9821 	if (event->prog && event->prog->type == BPF_PROG_TYPE_PERF_EVENT &&  in __perf_event_overflow()
9830 	event->pending_kill = POLL_IN;  in __perf_event_overflow()
9831 	if (events && atomic_dec_and_test(&event->event_limit)) {  in __perf_event_overflow()
9833 		event->pending_kill = POLL_HUP;  in __perf_event_overflow()
9837 	if (event->attr.sigtrap) {  in __perf_event_overflow()
9853 		if (!event->pending_work &&  in __perf_event_overflow()
9854 		    !task_work_add(current, &event->pending_task, notify_mode)) {  in __perf_event_overflow()
9855 			event->pending_work = pending_id;  in __perf_event_overflow()
9856 			local_inc(&event->ctx->nr_no_switch_fast);  in __perf_event_overflow()
9858 			event->pending_addr = 0;  in __perf_event_overflow()
9859 			if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR))  in __perf_event_overflow()
9860 				event->pending_addr = data->addr;  in __perf_event_overflow()
9862 		} else if (event->attr.exclude_kernel && valid_sample) {  in __perf_event_overflow()
9870 			 *  2. Events that can overflow again before the IRQ-  in __perf_event_overflow()
9873 			 *     check 32-bit hash of the current IP.  in __perf_event_overflow()
9875 			WARN_ON_ONCE(event->pending_work != pending_id);  in __perf_event_overflow()
9879 	READ_ONCE(event->overflow_handler)(event, data, regs);  in __perf_event_overflow()
9881 	if (*perf_event_fasync(event) && event->pending_kill) {  in __perf_event_overflow()
9882 		event->pending_wakeup = 1;  in __perf_event_overflow()
9883 		irq_work_queue(&event->pending_irq);  in __perf_event_overflow()
9908  * We directly increment event->count and keep a second value in
9909  * event->hw.period_left to count intervals. This period event
9910  * is kept in the range [-sample_period, 0] so that we can use the
9916 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_set_period()
9917 	u64 period = hwc->last_period;  in perf_swevent_set_period()
9921 	hwc->last_period = hwc->sample_period;  in perf_swevent_set_period()
9923 	old = local64_read(&hwc->period_left);  in perf_swevent_set_period()
9931 		val -= offset;  in perf_swevent_set_period()
9932 	} while (!local64_try_cmpxchg(&hwc->period_left, &old, val));  in perf_swevent_set_period()
9941 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_overflow()
9947 	if (hwc->interrupts == MAX_INTERRUPTS)  in perf_swevent_overflow()
9950 	for (; overflow; overflow--) {  in perf_swevent_overflow()
9955 			 * hwc->interrupts == MAX_INTERRUPTS.  in perf_swevent_overflow()
9967 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_event()
9969 	local64_add(nr, &event->count);  in perf_swevent_event()
9977 	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {  in perf_swevent_event()
9978 		data->period = nr;  in perf_swevent_event()
9981 		data->period = event->hw.last_period;  in perf_swevent_event()
9983 	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)  in perf_swevent_event()
9986 	if (local64_add_negative(nr, &hwc->period_left))  in perf_swevent_event()
9995 	if (event->hw.state & PERF_HES_STOPPED)  in perf_exclude_event()
9999 		if (event->attr.exclude_user && user_mode(regs))  in perf_exclude_event()
10002 		if (event->attr.exclude_kernel && !user_mode(regs))  in perf_exclude_event()
10015 	if (event->attr.type != type)  in perf_swevent_match()
10018 	if (event->attr.config != event_id)  in perf_swevent_match()
10039 	return &hlist->heads[hash];  in __find_swevent_head()
10048 	hlist = rcu_dereference(swhash->swevent_hlist);  in find_swevent_head_rcu()
10060 	u32 event_id = event->attr.config;  in find_swevent_head()
10061 	u64 type = event->attr.type;  in find_swevent_head()
10068 	hlist = rcu_dereference_protected(swhash->swevent_hlist,  in find_swevent_head()
10069 					  lockdep_is_held(&event->ctx->lock));  in find_swevent_head()
10102 	return get_recursion_context(current->perf_recursion);  in perf_swevent_get_recursion_context()
10108 	put_recursion_context(current->perf_recursion, rctx);  in perf_swevent_put_recursion_context()
10145 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_add()
10149 		hwc->last_period = hwc->sample_period;  in perf_swevent_add()
10153 	hwc->state = !(flags & PERF_EF_START);  in perf_swevent_add()
10157 		return -EINVAL;  in perf_swevent_add()
10159 	hlist_add_head_rcu(&event->hlist_entry, head);  in perf_swevent_add()
10167 	hlist_del_rcu(&event->hlist_entry);  in perf_swevent_del()
10172 	event->hw.state = 0;  in perf_swevent_start()
10177 	event->hw.state = PERF_HES_STOPPED;  in perf_swevent_stop()
10184 	return rcu_dereference_protected(swhash->swevent_hlist,  in swevent_hlist_deref()
10185 					 lockdep_is_held(&swhash->hlist_mutex));  in swevent_hlist_deref()
10195 	RCU_INIT_POINTER(swhash->swevent_hlist, NULL);  in swevent_hlist_release()
10203 	mutex_lock(&swhash->hlist_mutex);  in swevent_hlist_put_cpu()
10205 	if (!--swhash->hlist_refcount)  in swevent_hlist_put_cpu()
10208 	mutex_unlock(&swhash->hlist_mutex);  in swevent_hlist_put_cpu()
10224 	mutex_lock(&swhash->hlist_mutex);  in swevent_hlist_get_cpu()
10231 			err = -ENOMEM;  in swevent_hlist_get_cpu()
10234 		rcu_assign_pointer(swhash->swevent_hlist, hlist);  in swevent_hlist_get_cpu()
10236 	swhash->hlist_refcount++;  in swevent_hlist_get_cpu()
10238 	mutex_unlock(&swhash->hlist_mutex);  in swevent_hlist_get_cpu()
10271 	u64 event_id = event->attr.config;  in sw_perf_event_destroy()
10273 	WARN_ON(event->parent);  in sw_perf_event_destroy()
10284 	u64 event_id = event->attr.config;  in perf_swevent_init()
10286 	if (event->attr.type != PERF_TYPE_SOFTWARE)  in perf_swevent_init()
10287 		return -ENOENT;  in perf_swevent_init()
10293 		return -EOPNOTSUPP;  in perf_swevent_init()
10297 		event->attr.type = perf_cpu_clock.type;  in perf_swevent_init()
10298 		return -ENOENT;  in perf_swevent_init()
10300 		event->attr.type = perf_task_clock.type;  in perf_swevent_init()
10301 		return -ENOENT;  in perf_swevent_init()
10308 		return -ENOENT;  in perf_swevent_init()
10310 	if (!event->parent) {  in perf_swevent_init()
10318 		event->destroy = sw_perf_event_destroy;  in perf_swevent_init()
10348 	if (event->attr.type != PERF_TYPE_TRACEPOINT)  in perf_tp_event_init()
10349 		return -ENOENT;  in perf_tp_event_init()
10355 		return -EOPNOTSUPP;  in perf_tp_event_init()
10361 	event->destroy = tp_perf_event_destroy;  in perf_tp_event_init()
10380 	void *record = data->raw->frag.data;  in perf_tp_filter_match()
10383 	if (event->parent)  in perf_tp_filter_match()
10384 		event = event->parent;  in perf_tp_filter_match()
10386 	if (likely(!event->filter) || filter_match_preds(event->filter, record))  in perf_tp_filter_match()
10395 	if (event->hw.state & PERF_HES_STOPPED)  in perf_tp_event_match()
10398 	 * If exclude_kernel, only trace user-space tracepoints (uprobes)  in perf_tp_event_match()
10400 	if (event->attr.exclude_kernel && !user_mode(regs))  in perf_tp_event_match()
10421 	perf_tp_event(call->event.type, count, raw_data, size, regs, head,  in perf_trace_run_bpf_submit()
10433 	if (event->attr.config != entry->type)  in __perf_tp_event_target_task()
10436 	if (event->attr.sigtrap)  in __perf_tp_event_target_task()
10451 	perf_event_groups_for_cpu_pmu(event, &ctx->pinned_groups, cpu, pmu) {  in perf_tp_event_target_task()
10457 	perf_event_groups_for_cpu_pmu(event, &ctx->flexible_groups, cpu, pmu) {  in perf_tp_event_target_task()
10488 			 * Here use the same on-stack perf_sample_data,  in perf_tp_event()
10489 			 * some members in data are event-specific and  in perf_tp_event()
10490 			 * need to be re-computed for different sweveents.  in perf_tp_event()
10491 			 * Re-initialize data->sample_flags safely to avoid  in perf_tp_event()
10493 			 * because data->sample_flags is set.  in perf_tp_event()
10508 		ctx = rcu_dereference(task->perf_event_ctxp);  in perf_tp_event()
10512 		raw_spin_lock(&ctx->lock);  in perf_tp_event()
10514 		raw_spin_unlock(&ctx->lock);  in perf_tp_event()
10541 	PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
10580 	if (event->attr.type != perf_kprobe.type)  in perf_kprobe_event_init()
10581 		return -ENOENT;  in perf_kprobe_event_init()
10584 		return -EACCES;  in perf_kprobe_event_init()
10590 		return -EOPNOTSUPP;  in perf_kprobe_event_init()
10592 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;  in perf_kprobe_event_init()
10597 	event->destroy = perf_kprobe_destroy;  in perf_kprobe_event_init()
10604 PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
10640 	if (event->attr.type != perf_uprobe.type)  in perf_uprobe_event_init()
10641 		return -ENOENT;  in perf_uprobe_event_init()
10644 		return -EACCES;  in perf_uprobe_event_init()
10650 		return -EOPNOTSUPP;  in perf_uprobe_event_init()
10652 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;  in perf_uprobe_event_init()
10653 	ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT;  in perf_uprobe_event_init()
10658 	event->destroy = perf_uprobe_destroy;  in perf_uprobe_event_init()
10668 	perf_pmu_register(&perf_kprobe, "kprobe", -1);  in perf_tp_register()
10671 	perf_pmu_register(&perf_uprobe, "uprobe", -1);  in perf_tp_register()
10686 	if (event->pmu == &perf_tracepoint)  in perf_event_is_tracing()
10689 	if (event->pmu == &perf_kprobe)  in perf_event_is_tracing()
10693 	if (event->pmu == &perf_uprobe)  in perf_event_is_tracing()
10707 	is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE;  in perf_event_set_bpf_prog()
10708 	is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE;  in perf_event_set_bpf_prog()
10709 	is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;  in perf_event_set_bpf_prog()
10710 	is_syscall_tp = is_syscall_trace_event(event->tp_event);  in perf_event_set_bpf_prog()
10713 		return -EINVAL;  in perf_event_set_bpf_prog()
10715 	if (((is_kprobe || is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) ||  in perf_event_set_bpf_prog()
10716 	    (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) ||  in perf_event_set_bpf_prog()
10717 	    (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT))  in perf_event_set_bpf_prog()
10718 		return -EINVAL;  in perf_event_set_bpf_prog()
10720 	if (prog->type == BPF_PROG_TYPE_KPROBE && prog->sleepable && !is_uprobe)  in perf_event_set_bpf_prog()
10722 		return -EINVAL;  in perf_event_set_bpf_prog()
10725 	if (prog->kprobe_override && !is_kprobe)  in perf_event_set_bpf_prog()
10726 		return -EINVAL;  in perf_event_set_bpf_prog()
10729 		int off = trace_event_get_offsets(event->tp_event);  in perf_event_set_bpf_prog()
10731 		if (prog->aux->max_ctx_offset > off)  in perf_event_set_bpf_prog()
10732 			return -EACCES;  in perf_event_set_bpf_prog()
10760 	return -ENOENT;  in perf_event_set_bpf_prog()
10774 	perf_sample_data_init(&sample, bp->attr.bp_addr, 0);  in perf_bp_event()
10776 	if (!bp->hw.state && !perf_exclude_event(bp, regs))  in perf_bp_event()
10787 	int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu);  in perf_addr_filter_new()
10794 	INIT_LIST_HEAD(&filter->entry);  in perf_addr_filter_new()
10795 	list_add_tail(&filter->entry, filters);  in perf_addr_filter_new()
10805 		path_put(&filter->path);  in free_filters_list()
10806 		list_del(&filter->entry);  in free_filters_list()
10824 	if (event->parent)  in perf_addr_filters_splice()
10827 	raw_spin_lock_irqsave(&event->addr_filters.lock, flags);  in perf_addr_filters_splice()
10829 	list_splice_init(&event->addr_filters.list, &list);  in perf_addr_filters_splice()
10831 		list_splice(head, &event->addr_filters.list);  in perf_addr_filters_splice()
10833 	raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags);  in perf_addr_filters_splice()
10851 		if (!vma->vm_file)  in perf_addr_filter_apply()
10866 	struct task_struct *task = READ_ONCE(event->ctx->task);  in perf_event_addr_filters_apply()
10873 	 * We may observe TASK_TOMBSTONE, which means that the event tear-down  in perf_event_addr_filters_apply()
10879 	if (ifh->nr_file_filters) {  in perf_event_addr_filters_apply()
10887 	raw_spin_lock_irqsave(&ifh->lock, flags);  in perf_event_addr_filters_apply()
10888 	list_for_each_entry(filter, &ifh->list, entry) {  in perf_event_addr_filters_apply()
10889 		if (filter->path.dentry) {  in perf_event_addr_filters_apply()
10894 			event->addr_filter_ranges[count].start = 0;  in perf_event_addr_filters_apply()
10895 			event->addr_filter_ranges[count].size = 0;  in perf_event_addr_filters_apply()
10897 			perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]);  in perf_event_addr_filters_apply()
10899 			event->addr_filter_ranges[count].start = filter->offset;  in perf_event_addr_filters_apply()
10900 			event->addr_filter_ranges[count].size  = filter->size;  in perf_event_addr_filters_apply()
10906 	event->addr_filters_gen++;  in perf_event_addr_filters_apply()
10907 	raw_spin_unlock_irqrestore(&ifh->lock, flags);  in perf_event_addr_filters_apply()
10909 	if (ifh->nr_file_filters) {  in perf_event_addr_filters_apply()
10939 	IF_ACT_NONE = -1,
10978 	int ret = -EINVAL;  in perf_event_parse_addr_filter()
10982 		return -ENOMEM;  in perf_event_parse_addr_filter()
10990 		ret = -EINVAL;  in perf_event_parse_addr_filter()
11010 			filter->action = actions[token];  in perf_event_parse_addr_filter()
11025 			ret = kstrtoul(args[0].from, 0, &filter->offset);  in perf_event_parse_addr_filter()
11031 				ret = kstrtoul(args[1].from, 0, &filter->size);  in perf_event_parse_addr_filter()
11042 					ret = -ENOMEM;  in perf_event_parse_addr_filter()
11060 			ret = -EINVAL;  in perf_event_parse_addr_filter()
11063 			 * ACTION "filter" must have a non-zero length region  in perf_event_parse_addr_filter()
11066 			if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER &&  in perf_event_parse_addr_filter()
11067 			    !filter->size)  in perf_event_parse_addr_filter()
11075 				 * For now, we only support file-based filters  in perf_event_parse_addr_filter()
11076 				 * in per-task events; doing so for CPU-wide  in perf_event_parse_addr_filter()
11082 				ret = -EOPNOTSUPP;  in perf_event_parse_addr_filter()
11083 				if (!event->ctx->task)  in perf_event_parse_addr_filter()
11088 						&filter->path);  in perf_event_parse_addr_filter()
11092 				ret = -EINVAL;  in perf_event_parse_addr_filter()
11093 				if (!filter->path.dentry ||  in perf_event_parse_addr_filter()
11094 				    !S_ISREG(d_inode(filter->path.dentry)  in perf_event_parse_addr_filter()
11095 					     ->i_mode))  in perf_event_parse_addr_filter()
11098 				event->addr_filters.nr_file_filters++;  in perf_event_parse_addr_filter()
11136 	lockdep_assert_held(&event->ctx->mutex);  in perf_event_set_addr_filter()
11138 	if (WARN_ON_ONCE(event->parent))  in perf_event_set_addr_filter()
11139 		return -EINVAL;  in perf_event_set_addr_filter()
11145 	ret = event->pmu->addr_filters_validate(&filters);  in perf_event_set_addr_filter()
11161 	event->addr_filters.nr_file_filters = 0;  in perf_event_set_addr_filter()
11168 	int ret = -EINVAL;  in perf_event_set_filter()
11177 		struct perf_event_context *ctx = event->ctx;  in perf_event_set_filter()
11182 		 * the tracepoint muck will deadlock against ctx->mutex, but  in perf_event_set_filter()
11184 		 * temporarily drop ctx->mutex. As per perf_event_ctx_lock() we  in perf_event_set_filter()
11190 		mutex_unlock(&ctx->mutex);  in perf_event_set_filter()
11191 		ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);  in perf_event_set_filter()
11192 		mutex_lock(&ctx->mutex);  in perf_event_set_filter()
11216 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_swevent_hrtimer()
11219 	event->pmu->read(event);  in perf_swevent_hrtimer()
11221 	perf_sample_data_init(&data, 0, event->hw.last_period);  in perf_swevent_hrtimer()
11225 		if (!(event->attr.exclude_idle && is_idle_task(current)))  in perf_swevent_hrtimer()
11230 	period = max_t(u64, 10000, event->hw.sample_period);  in perf_swevent_hrtimer()
11238 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_start_hrtimer()
11244 	period = local64_read(&hwc->period_left);  in perf_swevent_start_hrtimer()
11249 		local64_set(&hwc->period_left, 0);  in perf_swevent_start_hrtimer()
11251 		period = max_t(u64, 10000, hwc->sample_period);  in perf_swevent_start_hrtimer()
11253 	hrtimer_start(&hwc->hrtimer, ns_to_ktime(period),  in perf_swevent_start_hrtimer()
11259 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_cancel_hrtimer()
11262 		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);  in perf_swevent_cancel_hrtimer()
11263 		local64_set(&hwc->period_left, ktime_to_ns(remaining));  in perf_swevent_cancel_hrtimer()
11265 		hrtimer_cancel(&hwc->hrtimer);  in perf_swevent_cancel_hrtimer()
11271 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_init_hrtimer()
11276 	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);  in perf_swevent_init_hrtimer()
11277 	hwc->hrtimer.function = perf_swevent_hrtimer;  in perf_swevent_init_hrtimer()
11280 	 * Since hrtimers have a fixed rate, we can do a static freq->period  in perf_swevent_init_hrtimer()
11283 	if (event->attr.freq) {  in perf_swevent_init_hrtimer()
11284 		long freq = event->attr.sample_freq;  in perf_swevent_init_hrtimer()
11286 		event->attr.sample_period = NSEC_PER_SEC / freq;  in perf_swevent_init_hrtimer()
11287 		hwc->sample_period = event->attr.sample_period;  in perf_swevent_init_hrtimer()
11288 		local64_set(&hwc->period_left, hwc->sample_period);  in perf_swevent_init_hrtimer()
11289 		hwc->last_period = hwc->sample_period;  in perf_swevent_init_hrtimer()
11290 		event->attr.freq = 0;  in perf_swevent_init_hrtimer()
11304 	prev = local64_xchg(&event->hw.prev_count, now);  in cpu_clock_event_update()
11305 	local64_add(now - prev, &event->count);  in cpu_clock_event_update()
11310 	local64_set(&event->hw.prev_count, local_clock());  in cpu_clock_event_start()
11341 	if (event->attr.type != perf_cpu_clock.type)  in cpu_clock_event_init()
11342 		return -ENOENT;  in cpu_clock_event_init()
11344 	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)  in cpu_clock_event_init()
11345 		return -ENOENT;  in cpu_clock_event_init()
11351 		return -EOPNOTSUPP;  in cpu_clock_event_init()
11381 	prev = local64_xchg(&event->hw.prev_count, now);  in task_clock_event_update()
11382 	delta = now - prev;  in task_clock_event_update()
11383 	local64_add(delta, &event->count);  in task_clock_event_update()
11388 	local64_set(&event->hw.prev_count, event->ctx->time);  in task_clock_event_start()
11395 	task_clock_event_update(event, event->ctx->time);  in task_clock_event_stop()
11415 	u64 delta = now - event->ctx->timestamp;  in task_clock_event_read()
11416 	u64 time = event->ctx->time + delta;  in task_clock_event_read()
11423 	if (event->attr.type != perf_task_clock.type)  in task_clock_event_init()
11424 		return -ENOENT;  in task_clock_event_init()
11426 	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)  in task_clock_event_init()
11427 		return -ENOENT;  in task_clock_event_init()
11433 		return -EOPNOTSUPP;  in task_clock_event_init()
11516 	free_percpu(pmu->cpu_pmu_context);  in free_pmu_context()
11528 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->nr_addr_filters);  in nr_addr_filters_show()
11539 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->type);  in type_show()
11550 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->hrtimer_interval_ms);  in perf_event_mux_interval_ms_show()
11568 		return -EINVAL;  in perf_event_mux_interval_ms_store()
11571 	if (timer == pmu->hrtimer_interval_ms)  in perf_event_mux_interval_ms_store()
11575 	pmu->hrtimer_interval_ms = timer;  in perf_event_mux_interval_ms_store()
11581 		cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu);  in perf_event_mux_interval_ms_store()
11582 		cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);  in perf_event_mux_interval_ms_store()
11633 	struct cpumask *mask = perf_scope_cpumask(pmu->scope);  in cpumask_show()
11655 	if (n == 2 && !pmu->nr_addr_filters)  in pmu_dev_is_visible()
11659 	if (n == 3 && pmu->scope == PERF_PMU_SCOPE_NONE)  in pmu_dev_is_visible()
11662 	return a->mode;  in pmu_dev_is_visible()
11688 	int ret = -ENOMEM;  in pmu_dev_alloc()
11690 	pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);  in pmu_dev_alloc()
11691 	if (!pmu->dev)  in pmu_dev_alloc()
11694 	pmu->dev->groups = pmu->attr_groups;  in pmu_dev_alloc()
11695 	device_initialize(pmu->dev);  in pmu_dev_alloc()
11697 	dev_set_drvdata(pmu->dev, pmu);  in pmu_dev_alloc()
11698 	pmu->dev->bus = &pmu_bus;  in pmu_dev_alloc()
11699 	pmu->dev->parent = pmu->parent;  in pmu_dev_alloc()
11700 	pmu->dev->release = pmu_dev_release;  in pmu_dev_alloc()
11702 	ret = dev_set_name(pmu->dev, "%s", pmu->name);  in pmu_dev_alloc()
11706 	ret = device_add(pmu->dev);  in pmu_dev_alloc()
11710 	if (pmu->attr_update) {  in pmu_dev_alloc()
11711 		ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update);  in pmu_dev_alloc()
11720 	device_del(pmu->dev);  in pmu_dev_alloc()
11723 	put_device(pmu->dev);  in pmu_dev_alloc()
11735 	ret = -ENOMEM;  in perf_pmu_register()
11736 	pmu->pmu_disable_count = alloc_percpu(int);  in perf_pmu_register()
11737 	if (!pmu->pmu_disable_count)  in perf_pmu_register()
11740 	pmu->type = -1;  in perf_pmu_register()
11742 		ret = -EINVAL;  in perf_pmu_register()
11746 …if (WARN_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE, "Can not register a pmu with an invalid scope.\n")…  in perf_pmu_register()
11747 		ret = -EINVAL;  in perf_pmu_register()
11751 	pmu->name = name;  in perf_pmu_register()
11763 	pmu->type = type;  in perf_pmu_register()
11765 	if (pmu_bus_running && !pmu->dev) {  in perf_pmu_register()
11771 	ret = -ENOMEM;  in perf_pmu_register()
11772 	pmu->cpu_pmu_context = alloc_percpu(struct perf_cpu_pmu_context);  in perf_pmu_register()
11773 	if (!pmu->cpu_pmu_context)  in perf_pmu_register()
11779 		cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu);  in perf_pmu_register()
11780 		__perf_init_event_pmu_context(&cpc->epc, pmu);  in perf_pmu_register()
11784 	if (!pmu->start_txn) {  in perf_pmu_register()
11785 		if (pmu->pmu_enable) {  in perf_pmu_register()
11791 			pmu->start_txn  = perf_pmu_start_txn;  in perf_pmu_register()
11792 			pmu->commit_txn = perf_pmu_commit_txn;  in perf_pmu_register()
11793 			pmu->cancel_txn = perf_pmu_cancel_txn;  in perf_pmu_register()
11795 			pmu->start_txn  = perf_pmu_nop_txn;  in perf_pmu_register()
11796 			pmu->commit_txn = perf_pmu_nop_int;  in perf_pmu_register()
11797 			pmu->cancel_txn = perf_pmu_nop_void;  in perf_pmu_register()
11801 	if (!pmu->pmu_enable) {  in perf_pmu_register()
11802 		pmu->pmu_enable  = perf_pmu_nop_void;  in perf_pmu_register()
11803 		pmu->pmu_disable = perf_pmu_nop_void;  in perf_pmu_register()
11806 	if (!pmu->check_period)  in perf_pmu_register()
11807 		pmu->check_period = perf_event_nop_int;  in perf_pmu_register()
11809 	if (!pmu->event_idx)  in perf_pmu_register()
11810 		pmu->event_idx = perf_event_idx_default;  in perf_pmu_register()
11812 	list_add_rcu(&pmu->entry, &pmus);  in perf_pmu_register()
11813 	atomic_set(&pmu->exclusive_cnt, 0);  in perf_pmu_register()
11821 	if (pmu->dev && pmu->dev != PMU_NULL_DEV) {  in perf_pmu_register()
11822 		device_del(pmu->dev);  in perf_pmu_register()
11823 		put_device(pmu->dev);  in perf_pmu_register()
11827 	idr_remove(&pmu_idr, pmu->type);  in perf_pmu_register()
11830 	free_percpu(pmu->pmu_disable_count);  in perf_pmu_register()
11838 	list_del_rcu(&pmu->entry);  in perf_pmu_unregister()
11847 	free_percpu(pmu->pmu_disable_count);  in perf_pmu_unregister()
11848 	idr_remove(&pmu_idr, pmu->type);  in perf_pmu_unregister()
11849 	if (pmu_bus_running && pmu->dev && pmu->dev != PMU_NULL_DEV) {  in perf_pmu_unregister()
11850 		if (pmu->nr_addr_filters)  in perf_pmu_unregister()
11851 			device_remove_file(pmu->dev, &dev_attr_nr_addr_filters);  in perf_pmu_unregister()
11852 		device_del(pmu->dev);  in perf_pmu_unregister()
11853 		put_device(pmu->dev);  in perf_pmu_unregister()
11862 	return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) ||  in has_extended_regs()
11863 	       (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK);  in has_extended_regs()
11871 	if (!try_module_get(pmu->module))  in perf_try_init_event()
11872 		return -ENODEV;  in perf_try_init_event()
11875 	 * A number of pmu->event_init() methods iterate the sibling_list to,  in perf_try_init_event()
11877 	 * if this is a sibling event, acquire the ctx->mutex to protect  in perf_try_init_event()
11880 	if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) {  in perf_try_init_event()
11882 		 * This ctx->mutex can nest when we're called through  in perf_try_init_event()
11885 		ctx = perf_event_ctx_lock_nested(event->group_leader,  in perf_try_init_event()
11890 	event->pmu = pmu;  in perf_try_init_event()
11891 	ret = pmu->event_init(event);  in perf_try_init_event()
11894 		perf_event_ctx_unlock(event->group_leader, ctx);  in perf_try_init_event()
11897 		if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) &&  in perf_try_init_event()
11899 			ret = -EOPNOTSUPP;  in perf_try_init_event()
11901 		if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE &&  in perf_try_init_event()
11903 			ret = -EINVAL;  in perf_try_init_event()
11905 		if (pmu->scope != PERF_PMU_SCOPE_NONE && event->cpu >= 0) {  in perf_try_init_event()
11906 			const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(pmu->scope, event->cpu);  in perf_try_init_event()
11907 			struct cpumask *pmu_cpumask = perf_scope_cpumask(pmu->scope);  in perf_try_init_event()
11913 					ret = -ENODEV;  in perf_try_init_event()
11915 					event->event_caps |= PERF_EV_CAP_READ_SCOPE;  in perf_try_init_event()
11917 				ret = -ENODEV;  in perf_try_init_event()
11921 		if (ret && event->destroy)  in perf_try_init_event()
11922 			event->destroy(event);  in perf_try_init_event()
11926 		module_put(pmu->module);  in perf_try_init_event()
11940 	 * Save original type before calling pmu->event_init() since certain  in perf_init_event()
11941 	 * pmus overwrites event->attr.type to forward event to another pmu.  in perf_init_event()
11943 	event->orig_type = event->attr.type;  in perf_init_event()
11946 	if (event->parent && event->parent->pmu) {  in perf_init_event()
11947 		pmu = event->parent->pmu;  in perf_init_event()
11957 	type = event->attr.type;  in perf_init_event()
11959 		type = event->attr.config >> PERF_PMU_TYPE_SHIFT;  in perf_init_event()
11964 			event->attr.config &= PERF_HW_EVENT_MASK;  in perf_init_event()
11973 		if (event->attr.type != type && type != PERF_TYPE_RAW &&  in perf_init_event()
11974 		    !(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE))  in perf_init_event()
11978 		if (ret == -ENOENT && event->attr.type != type && !extended_type) {  in perf_init_event()
11979 			type = event->attr.type;  in perf_init_event()
11994 		if (ret != -ENOENT) {  in perf_init_event()
12000 	pmu = ERR_PTR(-ENOENT);  in perf_init_event()
12009 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);  in attach_sb_event()
12011 	raw_spin_lock(&pel->lock);  in attach_sb_event()
12012 	list_add_rcu(&event->sb_list, &pel->list);  in attach_sb_event()
12013 	raw_spin_unlock(&pel->lock);  in attach_sb_event()
12017  * We keep a list of all !task (and therefore per-cpu) events
12018  * that need to receive side-band records.
12020  * This avoids having to scan all the various PMU per-cpu contexts
12054 	if (event->parent)  in account_event()
12057 	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))  in account_event()
12059 	if (event->attr.mmap || event->attr.mmap_data)  in account_event()
12061 	if (event->attr.build_id)  in account_event()
12063 	if (event->attr.comm)  in account_event()
12065 	if (event->attr.namespaces)  in account_event()
12067 	if (event->attr.cgroup)  in account_event()
12069 	if (event->attr.task)  in account_event()
12071 	if (event->attr.freq)  in account_event()
12073 	if (event->attr.context_switch) {  in account_event()
12081 	if (event->attr.ksymbol)  in account_event()
12083 	if (event->attr.bpf_event)  in account_event()
12085 	if (event->attr.text_poke)  in account_event()
12109 		 * increments to by-pass the mutex.  in account_event()
12133 	long err = -EINVAL;  in perf_event_alloc()
12137 		if (!task || cpu != -1)  in perf_event_alloc()
12138 			return ERR_PTR(-EINVAL);  in perf_event_alloc()
12140 	if (attr->sigtrap && !task) {  in perf_event_alloc()
12142 		return ERR_PTR(-EINVAL);  in perf_event_alloc()
12145 	node = (cpu >= 0) ? cpu_to_node(cpu) : -1;  in perf_event_alloc()
12149 		return ERR_PTR(-ENOMEM);  in perf_event_alloc()
12158 	mutex_init(&event->child_mutex);  in perf_event_alloc()
12159 	INIT_LIST_HEAD(&event->child_list);  in perf_event_alloc()
12161 	INIT_LIST_HEAD(&event->event_entry);  in perf_event_alloc()
12162 	INIT_LIST_HEAD(&event->sibling_list);  in perf_event_alloc()
12163 	INIT_LIST_HEAD(&event->active_list);  in perf_event_alloc()
12165 	INIT_LIST_HEAD(&event->rb_entry);  in perf_event_alloc()
12166 	INIT_LIST_HEAD(&event->active_entry);  in perf_event_alloc()
12167 	INIT_LIST_HEAD(&event->addr_filters.list);  in perf_event_alloc()
12168 	INIT_HLIST_NODE(&event->hlist_entry);  in perf_event_alloc()
12171 	init_waitqueue_head(&event->waitq);  in perf_event_alloc()
12172 	init_irq_work(&event->pending_irq, perf_pending_irq);  in perf_event_alloc()
12173 	event->pending_disable_irq = IRQ_WORK_INIT_HARD(perf_pending_disable);  in perf_event_alloc()
12174 	init_task_work(&event->pending_task, perf_pending_task);  in perf_event_alloc()
12175 	rcuwait_init(&event->pending_work_wait);  in perf_event_alloc()
12177 	mutex_init(&event->mmap_mutex);  in perf_event_alloc()
12178 	raw_spin_lock_init(&event->addr_filters.lock);  in perf_event_alloc()
12180 	atomic_long_set(&event->refcount, 1);  in perf_event_alloc()
12181 	event->cpu		= cpu;  in perf_event_alloc()
12182 	event->attr		= *attr;  in perf_event_alloc()
12183 	event->group_leader	= group_leader;  in perf_event_alloc()
12184 	event->pmu		= NULL;  in perf_event_alloc()
12185 	event->oncpu		= -1;  in perf_event_alloc()
12187 	event->parent		= parent_event;  in perf_event_alloc()
12189 	event->ns		= get_pid_ns(task_active_pid_ns(current));  in perf_event_alloc()
12190 	event->id		= atomic64_inc_return(&perf_event_id);  in perf_event_alloc()
12192 	event->state		= PERF_EVENT_STATE_INACTIVE;  in perf_event_alloc()
12195 		event->event_caps = parent_event->event_caps;  in perf_event_alloc()
12198 		event->attach_state = PERF_ATTACH_TASK;  in perf_event_alloc()
12204 		event->hw.target = get_task_struct(task);  in perf_event_alloc()
12207 	event->clock = &local_clock;  in perf_event_alloc()
12209 		event->clock = parent_event->clock;  in perf_event_alloc()
12212 		overflow_handler = parent_event->overflow_handler;  in perf_event_alloc()
12213 		context = parent_event->overflow_handler_context;  in perf_event_alloc()
12215 		if (parent_event->prog) {  in perf_event_alloc()
12216 			struct bpf_prog *prog = parent_event->prog;  in perf_event_alloc()
12219 			event->prog = prog;  in perf_event_alloc()
12225 		event->overflow_handler	= overflow_handler;  in perf_event_alloc()
12226 		event->overflow_handler_context = context;  in perf_event_alloc()
12228 		event->overflow_handler = perf_event_output_backward;  in perf_event_alloc()
12229 		event->overflow_handler_context = NULL;  in perf_event_alloc()
12231 		event->overflow_handler = perf_event_output_forward;  in perf_event_alloc()
12232 		event->overflow_handler_context = NULL;  in perf_event_alloc()
12239 	hwc = &event->hw;  in perf_event_alloc()
12240 	hwc->sample_period = attr->sample_period;  in perf_event_alloc()
12241 	if (attr->freq && attr->sample_freq)  in perf_event_alloc()
12242 		hwc->sample_period = 1;  in perf_event_alloc()
12243 	hwc->last_period = hwc->sample_period;  in perf_event_alloc()
12245 	local64_set(&hwc->period_left, hwc->sample_period);  in perf_event_alloc()
12250 	 * collect per-thread samples.  in perf_event_alloc()
12253 	if (has_inherit_and_sample_read(attr) && !(attr->sample_type & PERF_SAMPLE_TID))  in perf_event_alloc()
12257 		event->attr.branch_sample_type = 0;  in perf_event_alloc()
12266 	 * Disallow uncore-task events. Similarly, disallow uncore-cgroup  in perf_event_alloc()
12270 	if (pmu->task_ctx_nr == perf_invalid_context && (task || cgroup_fd != -1)) {  in perf_event_alloc()
12271 		err = -EINVAL;  in perf_event_alloc()
12275 	if (event->attr.aux_output &&  in perf_event_alloc()
12276 	    !(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT)) {  in perf_event_alloc()
12277 		err = -EOPNOTSUPP;  in perf_event_alloc()
12281 	if (cgroup_fd != -1) {  in perf_event_alloc()
12292 		event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters,  in perf_event_alloc()
12295 		if (!event->addr_filter_ranges) {  in perf_event_alloc()
12296 			err = -ENOMEM;  in perf_event_alloc()
12304 		if (event->parent) {  in perf_event_alloc()
12307 			raw_spin_lock_irq(&ifh->lock);  in perf_event_alloc()
12308 			memcpy(event->addr_filter_ranges,  in perf_event_alloc()
12309 			       event->parent->addr_filter_ranges,  in perf_event_alloc()
12310 			       pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range));  in perf_event_alloc()
12311 			raw_spin_unlock_irq(&ifh->lock);  in perf_event_alloc()
12315 		event->addr_filters_gen = 1;  in perf_event_alloc()
12318 	if (!event->parent) {  in perf_event_alloc()
12319 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {  in perf_event_alloc()
12320 			err = get_callchain_buffers(attr->sample_max_stack);  in perf_event_alloc()
12336 	if (!event->parent) {  in perf_event_alloc()
12337 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)  in perf_event_alloc()
12341 	kfree(event->addr_filter_ranges);  in perf_event_alloc()
12349 	if (event->destroy)  in perf_event_alloc()
12350 		event->destroy(event);  in perf_event_alloc()
12351 	module_put(pmu->module);  in perf_event_alloc()
12353 	if (event->hw.target)  in perf_event_alloc()
12354 		put_task_struct(event->hw.target);  in perf_event_alloc()
12355 	call_rcu(&event->rcu_head, free_event_rcu);  in perf_event_alloc()
12369 	ret = get_user(size, &uattr->size);  in perf_copy_attr()
12381 		if (ret == -E2BIG)  in perf_copy_attr()
12386 	attr->size = size;  in perf_copy_attr()
12388 	if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)  in perf_copy_attr()
12389 		return -EINVAL;  in perf_copy_attr()
12391 	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))  in perf_copy_attr()
12392 		return -EINVAL;  in perf_copy_attr()
12394 	if (attr->read_format & ~(PERF_FORMAT_MAX-1))  in perf_copy_attr()
12395 		return -EINVAL;  in perf_copy_attr()
12397 	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {  in perf_copy_attr()
12398 		u64 mask = attr->branch_sample_type;  in perf_copy_attr()
12401 		if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))  in perf_copy_attr()
12402 			return -EINVAL;  in perf_copy_attr()
12406 			return -EINVAL;  in perf_copy_attr()
12412 			if (!attr->exclude_kernel)  in perf_copy_attr()
12415 			if (!attr->exclude_user)  in perf_copy_attr()
12418 			if (!attr->exclude_hv)  in perf_copy_attr()
12423 			attr->branch_sample_type = mask;  in perf_copy_attr()
12433 	if (attr->sample_type & PERF_SAMPLE_REGS_USER) {  in perf_copy_attr()
12434 		ret = perf_reg_validate(attr->sample_regs_user);  in perf_copy_attr()
12439 	if (attr->sample_type & PERF_SAMPLE_STACK_USER) {  in perf_copy_attr()
12441 			return -ENOSYS;  in perf_copy_attr()
12448 		if (attr->sample_stack_user >= USHRT_MAX)  in perf_copy_attr()
12449 			return -EINVAL;  in perf_copy_attr()
12450 		else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))  in perf_copy_attr()
12451 			return -EINVAL;  in perf_copy_attr()
12454 	if (!attr->sample_max_stack)  in perf_copy_attr()
12455 		attr->sample_max_stack = sysctl_perf_event_max_stack;  in perf_copy_attr()
12457 	if (attr->sample_type & PERF_SAMPLE_REGS_INTR)  in perf_copy_attr()
12458 		ret = perf_reg_validate(attr->sample_regs_intr);  in perf_copy_attr()
12461 	if (attr->sample_type & PERF_SAMPLE_CGROUP)  in perf_copy_attr()
12462 		return -EINVAL;  in perf_copy_attr()
12464 	if ((attr->sample_type & PERF_SAMPLE_WEIGHT) &&  in perf_copy_attr()
12465 	    (attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT))  in perf_copy_attr()
12466 		return -EINVAL;  in perf_copy_attr()
12468 	if (!attr->inherit && attr->inherit_thread)  in perf_copy_attr()
12469 		return -EINVAL;  in perf_copy_attr()
12471 	if (attr->remove_on_exec && attr->enable_on_exec)  in perf_copy_attr()
12472 		return -EINVAL;  in perf_copy_attr()
12474 	if (attr->sigtrap && !attr->remove_on_exec)  in perf_copy_attr()
12475 		return -EINVAL;  in perf_copy_attr()
12481 	put_user(sizeof(*attr), &uattr->size);  in perf_copy_attr()
12482 	ret = -E2BIG;  in perf_copy_attr()
12499 	int ret = -EINVAL;  in perf_event_set_output()
12502 		mutex_lock(&event->mmap_mutex);  in perf_event_set_output()
12511 	 * Don't allow cross-cpu buffers  in perf_event_set_output()
12513 	if (output_event->cpu != event->cpu)  in perf_event_set_output()
12517 	 * If its not a per-cpu rb, it must be the same task.  in perf_event_set_output()
12519 	if (output_event->cpu == -1 && output_event->hw.target != event->hw.target)  in perf_event_set_output()
12525 	if (output_event->clock != event->clock)  in perf_event_set_output()
12539 	    event->pmu != output_event->pmu)  in perf_event_set_output()
12544 	 * output_event is already on rb->event_list, and the list iteration  in perf_event_set_output()
12547 	 * observe !rb->mmap_count.  in perf_event_set_output()
12549 	mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex);  in perf_event_set_output()
12552 	if (atomic_read(&event->mmap_count))  in perf_event_set_output()
12562 		if (!atomic_read(&rb->mmap_count)) {  in perf_event_set_output()
12572 	mutex_unlock(&event->mmap_mutex);  in perf_event_set_output()
12574 		mutex_unlock(&output_event->mmap_mutex);  in perf_event_set_output()
12586 		event->clock = &ktime_get_mono_fast_ns;  in perf_event_set_clock()
12591 		event->clock = &ktime_get_raw_fast_ns;  in perf_event_set_clock()
12596 		event->clock = &ktime_get_real_ns;  in perf_event_set_clock()
12600 		event->clock = &ktime_get_boottime_ns;  in perf_event_set_clock()
12604 		event->clock = &ktime_get_clocktai_ns;  in perf_event_set_clock()
12608 		return -EINVAL;  in perf_event_set_clock()
12611 	if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI))  in perf_event_set_clock()
12612 		return -EINVAL;  in perf_event_set_clock()
12623 	if (attr->sigtrap) {  in perf_check_permission()
12629 		is_capable &= ns_capable(__task_cred(task)->user_ns, CAP_KILL);  in perf_check_permission()
12649  * sys_perf_event_open - open a performance event, associate it to a task/cpu
12674 	int cgroup_fd = -1;  in SYSCALL_DEFINE5()
12678 		return -EINVAL;  in SYSCALL_DEFINE5()
12697 			return -EACCES;  in SYSCALL_DEFINE5()
12702 			return -EINVAL;  in SYSCALL_DEFINE5()
12705 			return -EINVAL;  in SYSCALL_DEFINE5()
12728 	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))  in SYSCALL_DEFINE5()
12729 		return -EINVAL;  in SYSCALL_DEFINE5()
12738 	if (group_fd != -1) {  in SYSCALL_DEFINE5()
12742 		group_leader = fd_file(group)->private_data;  in SYSCALL_DEFINE5()
12749 	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {  in SYSCALL_DEFINE5()
12758 	    group_leader->attr.inherit != attr.inherit) {  in SYSCALL_DEFINE5()
12759 		err = -EINVAL;  in SYSCALL_DEFINE5()
12774 		if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {  in SYSCALL_DEFINE5()
12775 			err = -EOPNOTSUPP;  in SYSCALL_DEFINE5()
12784 	pmu = event->pmu;  in SYSCALL_DEFINE5()
12792 	if (pmu->task_ctx_nr == perf_sw_context)  in SYSCALL_DEFINE5()
12793 		event->event_caps |= PERF_EV_CAP_SOFTWARE;  in SYSCALL_DEFINE5()
12796 		err = down_read_interruptible(&task->signal->exec_update_lock);  in SYSCALL_DEFINE5()
12806 		err = -EACCES;  in SYSCALL_DEFINE5()
12820 	mutex_lock(&ctx->mutex);  in SYSCALL_DEFINE5()
12822 	if (ctx->task == TASK_TOMBSTONE) {  in SYSCALL_DEFINE5()
12823 		err = -ESRCH;  in SYSCALL_DEFINE5()
12834 		struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);  in SYSCALL_DEFINE5()
12836 		if (!cpuctx->online) {  in SYSCALL_DEFINE5()
12837 			err = -ENODEV;  in SYSCALL_DEFINE5()
12843 		err = -EINVAL;  in SYSCALL_DEFINE5()
12847 		 * becoming part of another group-sibling):  in SYSCALL_DEFINE5()
12849 		if (group_leader->group_leader != group_leader)  in SYSCALL_DEFINE5()
12853 		if (group_leader->clock != event->clock)  in SYSCALL_DEFINE5()
12861 		if (group_leader->cpu != event->cpu)  in SYSCALL_DEFINE5()
12867 		if (group_leader->ctx != ctx)  in SYSCALL_DEFINE5()
12889 			pmu = group_leader->pmu_ctx->pmu;  in SYSCALL_DEFINE5()
12892 			    (group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) {  in SYSCALL_DEFINE5()
12903 			    group_leader->pmu_ctx->pmu != pmu)  in SYSCALL_DEFINE5()
12916 	event->pmu_ctx = pmu_ctx;  in SYSCALL_DEFINE5()
12925 		err = -E2BIG;  in SYSCALL_DEFINE5()
12930 		err = -EINVAL;  in SYSCALL_DEFINE5()
12939 		err = -EBUSY;  in SYSCALL_DEFINE5()
12943 	WARN_ON_ONCE(ctx->parent_ctx);  in SYSCALL_DEFINE5()
12959 		put_pmu_ctx(group_leader->pmu_ctx);  in SYSCALL_DEFINE5()
12963 			put_pmu_ctx(sibling->pmu_ctx);  in SYSCALL_DEFINE5()
12970 		 * (through the sibling list, which is still in-tact), we can  in SYSCALL_DEFINE5()
12973 		 * By installing siblings first we NO-OP because they're not  in SYSCALL_DEFINE5()
12977 			sibling->pmu_ctx = pmu_ctx;  in SYSCALL_DEFINE5()
12980 			perf_install_in_context(ctx, sibling, sibling->cpu);  in SYSCALL_DEFINE5()
12988 		group_leader->pmu_ctx = pmu_ctx;  in SYSCALL_DEFINE5()
12991 		perf_install_in_context(ctx, group_leader, group_leader->cpu);  in SYSCALL_DEFINE5()
13003 	event->owner = current;  in SYSCALL_DEFINE5()
13005 	perf_install_in_context(ctx, event, event->cpu);  in SYSCALL_DEFINE5()
13008 	mutex_unlock(&ctx->mutex);  in SYSCALL_DEFINE5()
13011 		up_read(&task->signal->exec_update_lock);  in SYSCALL_DEFINE5()
13015 	mutex_lock(&current->perf_event_mutex);  in SYSCALL_DEFINE5()
13016 	list_add_tail(&event->owner_entry, &current->perf_event_list);  in SYSCALL_DEFINE5()
13017 	mutex_unlock(&current->perf_event_mutex);  in SYSCALL_DEFINE5()
13030 	put_pmu_ctx(event->pmu_ctx);  in SYSCALL_DEFINE5()
13031 	event->pmu_ctx = NULL; /* _free_event() */  in SYSCALL_DEFINE5()
13033 	mutex_unlock(&ctx->mutex);  in SYSCALL_DEFINE5()
13038 		up_read(&task->signal->exec_update_lock);  in SYSCALL_DEFINE5()
13076 	if (attr->aux_output)  in perf_event_create_kernel_counter()
13077 		return ERR_PTR(-EINVAL);  in perf_event_create_kernel_counter()
13080 				 overflow_handler, context, -1);  in perf_event_create_kernel_counter()
13087 	event->owner = TASK_TOMBSTONE;  in perf_event_create_kernel_counter()
13088 	pmu = event->pmu;  in perf_event_create_kernel_counter()
13090 	if (pmu->task_ctx_nr == perf_sw_context)  in perf_event_create_kernel_counter()
13091 		event->event_caps |= PERF_EV_CAP_SOFTWARE;  in perf_event_create_kernel_counter()
13102 	WARN_ON_ONCE(ctx->parent_ctx);  in perf_event_create_kernel_counter()
13103 	mutex_lock(&ctx->mutex);  in perf_event_create_kernel_counter()
13104 	if (ctx->task == TASK_TOMBSTONE) {  in perf_event_create_kernel_counter()
13105 		err = -ESRCH;  in perf_event_create_kernel_counter()
13114 	event->pmu_ctx = pmu_ctx;  in perf_event_create_kernel_counter()
13125 		if (!cpuctx->online) {  in perf_event_create_kernel_counter()
13126 			err = -ENODEV;  in perf_event_create_kernel_counter()
13132 		err = -EBUSY;  in perf_event_create_kernel_counter()
13136 	perf_install_in_context(ctx, event, event->cpu);  in perf_event_create_kernel_counter()
13138 	mutex_unlock(&ctx->mutex);  in perf_event_create_kernel_counter()
13144 	event->pmu_ctx = NULL; /* _free_event() */  in perf_event_create_kernel_counter()
13146 	mutex_unlock(&ctx->mutex);  in perf_event_create_kernel_counter()
13165 		put_pmu_ctx(event->pmu_ctx);  in __perf_pmu_remove()
13166 		list_add(&event->migrate_entry, events);  in __perf_pmu_remove()
13170 			put_pmu_ctx(sibling->pmu_ctx);  in __perf_pmu_remove()
13171 			list_add(&sibling->migrate_entry, events);  in __perf_pmu_remove()
13181 	struct perf_event_context *old_ctx = event->ctx;  in __perf_pmu_install_event()
13185 	event->cpu = cpu;  in __perf_pmu_install_event()
13187 	event->pmu_ctx = epc;  in __perf_pmu_install_event()
13189 	if (event->state >= PERF_EVENT_STATE_OFF)  in __perf_pmu_install_event()
13190 		event->state = PERF_EVENT_STATE_INACTIVE;  in __perf_pmu_install_event()
13194 	 * Now that event->ctx is updated and visible, put the old ctx.  in __perf_pmu_install_event()
13205 	 * Re-instate events in 2 passes.  in __perf_pmu_install()
13213 		if (event->group_leader == event)  in __perf_pmu_install()
13216 		list_del(&event->migrate_entry);  in __perf_pmu_install()
13225 		list_del(&event->migrate_entry);  in __perf_pmu_install()
13236 	 * Since per-cpu context is persistent, no need to grab an extra  in perf_pmu_migrate_context()
13239 	src_ctx = &per_cpu_ptr(&perf_cpu_context, src_cpu)->ctx;  in perf_pmu_migrate_context()
13240 	dst_ctx = &per_cpu_ptr(&perf_cpu_context, dst_cpu)->ctx;  in perf_pmu_migrate_context()
13246 	mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex);  in perf_pmu_migrate_context()
13248 	__perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->pinned_groups, &events);  in perf_pmu_migrate_context()
13249 	__perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->flexible_groups, &events);  in perf_pmu_migrate_context()
13253 		 * Wait for the events to quiesce before re-instating them.  in perf_pmu_migrate_context()
13260 	mutex_unlock(&dst_ctx->mutex);  in perf_pmu_migrate_context()
13261 	mutex_unlock(&src_ctx->mutex);  in perf_pmu_migrate_context()
13267 	struct perf_event *parent_event = child_event->parent;  in sync_child_event()
13270 	if (child_event->attr.inherit_stat) {  in sync_child_event()
13271 		struct task_struct *task = child_event->ctx->task;  in sync_child_event()
13282 	atomic64_add(child_val, &parent_event->child_count);  in sync_child_event()
13283 	atomic64_add(child_event->total_time_enabled,  in sync_child_event()
13284 		     &parent_event->child_total_time_enabled);  in sync_child_event()
13285 	atomic64_add(child_event->total_time_running,  in sync_child_event()
13286 		     &parent_event->child_total_time_running);  in sync_child_event()
13292 	struct perf_event *parent_event = event->parent;  in perf_event_exit_event()
13309 		mutex_lock(&parent_event->child_mutex);  in perf_event_exit_event()
13314 	raw_spin_lock_irq(&ctx->lock);  in perf_event_exit_event()
13315 	if (event->state > PERF_EVENT_STATE_EXIT)  in perf_event_exit_event()
13317 	raw_spin_unlock_irq(&ctx->lock);  in perf_event_exit_event()
13323 		mutex_unlock(&parent_event->child_mutex);  in perf_event_exit_event()
13351 	 * In order to reduce the amount of tricky in ctx tear-down, we hold  in perf_event_exit_task_context()
13360 	mutex_lock(&child_ctx->mutex);  in perf_event_exit_task_context()
13363 	 * In a single ctx::lock section, de-schedule the events and detach the  in perf_event_exit_task_context()
13367 	raw_spin_lock_irq(&child_ctx->lock);  in perf_event_exit_task_context()
13371 	 * Now that the context is inactive, destroy the task <-> ctx relation  in perf_event_exit_task_context()
13374 	RCU_INIT_POINTER(child->perf_event_ctxp, NULL);  in perf_event_exit_task_context()
13376 	WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE);  in perf_event_exit_task_context()
13380 	raw_spin_unlock_irq(&child_ctx->lock);  in perf_event_exit_task_context()
13392 	list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)  in perf_event_exit_task_context()
13395 	mutex_unlock(&child_ctx->mutex);  in perf_event_exit_task_context()
13410 	mutex_lock(&child->perf_event_mutex);  in perf_event_exit_task()
13411 	list_for_each_entry_safe(event, tmp, &child->perf_event_list,  in perf_event_exit_task()
13413 		list_del_init(&event->owner_entry);  in perf_event_exit_task()
13418 		 * we need to serialize on the owner->perf_event_mutex.  in perf_event_exit_task()
13420 		smp_store_release(&event->owner, NULL);  in perf_event_exit_task()
13422 	mutex_unlock(&child->perf_event_mutex);  in perf_event_exit_task()
13429 	 * child contexts and sets child->perf_event_ctxp[] to NULL.  in perf_event_exit_task()
13438 	struct perf_event *parent = event->parent;  in perf_free_event()
13443 	mutex_lock(&parent->child_mutex);  in perf_free_event()
13444 	list_del_init(&event->child_list);  in perf_free_event()
13445 	mutex_unlock(&parent->child_mutex);  in perf_free_event()
13449 	raw_spin_lock_irq(&ctx->lock);  in perf_free_event()
13452 	raw_spin_unlock_irq(&ctx->lock);  in perf_free_event()
13468 	ctx = rcu_access_pointer(task->perf_event_ctxp);  in perf_event_free_task()
13472 	mutex_lock(&ctx->mutex);  in perf_event_free_task()
13473 	raw_spin_lock_irq(&ctx->lock);  in perf_event_free_task()
13475 	 * Destroy the task <-> ctx relation and mark the context dead.  in perf_event_free_task()
13480 	RCU_INIT_POINTER(task->perf_event_ctxp, NULL);  in perf_event_free_task()
13481 	WRITE_ONCE(ctx->task, TASK_TOMBSTONE);  in perf_event_free_task()
13483 	raw_spin_unlock_irq(&ctx->lock);  in perf_event_free_task()
13486 	list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry)  in perf_event_free_task()
13489 	mutex_unlock(&ctx->mutex);  in perf_event_free_task()
13500 	 * _free_event()'s put_task_struct(event->hw.target) will be a  in perf_event_free_task()
13501 	 * use-after-free.  in perf_event_free_task()
13505 	wait_var_event(&ctx->refcount, refcount_read(&ctx->refcount) == 1);  in perf_event_free_task()
13511 	WARN_ON_ONCE(task->perf_event_ctxp);  in perf_event_delayed_put()
13518 		return ERR_PTR(-EBADF);  in perf_event_get()
13520 	if (file->f_op != &perf_fops) {  in perf_event_get()
13522 		return ERR_PTR(-EBADF);  in perf_event_get()
13530 	if (file->f_op != &perf_fops)  in perf_get_event()
13531 		return ERR_PTR(-EINVAL);  in perf_get_event()
13533 	return file->private_data;  in perf_get_event()
13539 		return ERR_PTR(-EINVAL);  in perf_event_attrs()
13541 	return &event->attr;  in perf_event_attrs()
13547 		return -EACCES;  in perf_allow_kernel()
13557  *  - valid pointer on success
13558  *  - NULL for orphaned events
13559  *  - IS_ERR() on error
13569 	enum perf_event_state parent_state = parent_event->state;  in inherit_event()
13580 	if (parent_event->parent)  in inherit_event()
13581 		parent_event = parent_event->parent;  in inherit_event()
13583 	child_event = perf_event_alloc(&parent_event->attr,  in inherit_event()
13584 					   parent_event->cpu,  in inherit_event()
13587 					   NULL, NULL, -1);  in inherit_event()
13591 	pmu_ctx = find_get_pmu_context(child_event->pmu, child_ctx, child_event);  in inherit_event()
13596 	child_event->pmu_ctx = pmu_ctx;  in inherit_event()
13599 	 * is_orphaned_event() and list_add_tail(&parent_event->child_list)  in inherit_event()
13604 	mutex_lock(&parent_event->child_mutex);  in inherit_event()
13606 	    !atomic_long_inc_not_zero(&parent_event->refcount)) {  in inherit_event()
13607 		mutex_unlock(&parent_event->child_mutex);  in inherit_event()
13621 		child_event->state = PERF_EVENT_STATE_INACTIVE;  in inherit_event()
13623 		child_event->state = PERF_EVENT_STATE_OFF;  in inherit_event()
13625 	if (parent_event->attr.freq) {  in inherit_event()
13626 		u64 sample_period = parent_event->hw.sample_period;  in inherit_event()
13627 		struct hw_perf_event *hwc = &child_event->hw;  in inherit_event()
13629 		hwc->sample_period = sample_period;  in inherit_event()
13630 		hwc->last_period   = sample_period;  in inherit_event()
13632 		local64_set(&hwc->period_left, sample_period);  in inherit_event()
13635 	child_event->ctx = child_ctx;  in inherit_event()
13636 	child_event->overflow_handler = parent_event->overflow_handler;  in inherit_event()
13637 	child_event->overflow_handler_context  in inherit_event()
13638 		= parent_event->overflow_handler_context;  in inherit_event()
13649 	raw_spin_lock_irqsave(&child_ctx->lock, flags);  in inherit_event()
13651 	child_event->attach_state |= PERF_ATTACH_CHILD;  in inherit_event()
13652 	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);  in inherit_event()
13657 	list_add_tail(&child_event->child_list, &parent_event->child_list);  in inherit_event()
13658 	mutex_unlock(&parent_event->child_mutex);  in inherit_event()
13670  *  - 0 on success
13671  *  - <0 on error
13698 		if (sub->aux_event == parent_event && child_ctr &&  in inherit_group()
13700 			return -EINVAL;  in inherit_group()
13703 		leader->group_generation = parent_event->group_generation;  in inherit_group()
13708  * Creates the child task context and tries to inherit the event-group.
13715  *  - 0 on success
13716  *  - <0 on error
13727 	if (!event->attr.inherit ||  in inherit_task_group()
13728 	    (event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) ||  in inherit_task_group()
13730 	    (event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) {  in inherit_task_group()
13735 	child_ctx = child->perf_event_ctxp;  in inherit_task_group()
13745 			return -ENOMEM;  in inherit_task_group()
13747 		child->perf_event_ctxp = child_ctx;  in inherit_task_group()
13770 	if (likely(!parent->perf_event_ctxp))  in perf_event_init_context()
13783 	 * it non-NULL earlier, the only reason for it to become NULL  in perf_event_init_context()
13789 	 * Lock the parent list. No need to lock the child - not PID  in perf_event_init_context()
13792 	mutex_lock(&parent_ctx->mutex);  in perf_event_init_context()
13795 	 * We dont have to disable NMIs - we are only looking at  in perf_event_init_context()
13798 	perf_event_groups_for_each(event, &parent_ctx->pinned_groups) {  in perf_event_init_context()
13806 	 * We can't hold ctx->lock when iterating the ->flexible_group list due  in perf_event_init_context()
13810 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);  in perf_event_init_context()
13811 	parent_ctx->rotate_disable = 1;  in perf_event_init_context()
13812 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);  in perf_event_init_context()
13814 	perf_event_groups_for_each(event, &parent_ctx->flexible_groups) {  in perf_event_init_context()
13821 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);  in perf_event_init_context()
13822 	parent_ctx->rotate_disable = 0;  in perf_event_init_context()
13824 	child_ctx = child->perf_event_ctxp;  in perf_event_init_context()
13832 		 * parent_ctx->lock avoids it from being uncloned.  in perf_event_init_context()
13834 		cloned_ctx = parent_ctx->parent_ctx;  in perf_event_init_context()
13836 			child_ctx->parent_ctx = cloned_ctx;  in perf_event_init_context()
13837 			child_ctx->parent_gen = parent_ctx->parent_gen;  in perf_event_init_context()
13839 			child_ctx->parent_ctx = parent_ctx;  in perf_event_init_context()
13840 			child_ctx->parent_gen = parent_ctx->generation;  in perf_event_init_context()
13842 		get_ctx(child_ctx->parent_ctx);  in perf_event_init_context()
13845 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);  in perf_event_init_context()
13847 	mutex_unlock(&parent_ctx->mutex);  in perf_event_init_context()
13862 	memset(child->perf_recursion, 0, sizeof(child->perf_recursion));  in perf_event_init_task()
13863 	child->perf_event_ctxp = NULL;  in perf_event_init_task()
13864 	mutex_init(&child->perf_event_mutex);  in perf_event_init_task()
13865 	INIT_LIST_HEAD(&child->perf_event_list);  in perf_event_init_task()
13892 		mutex_init(&swhash->hlist_mutex);  in perf_event_init_all_cpus()
13900 		__perf_event_init_context(&cpuctx->ctx);  in perf_event_init_all_cpus()
13901 		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);  in perf_event_init_all_cpus()
13902 		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);  in perf_event_init_all_cpus()
13903 		cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask);  in perf_event_init_all_cpus()
13904 		cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default);  in perf_event_init_all_cpus()
13905 		cpuctx->heap = cpuctx->heap_default;  in perf_event_init_all_cpus()
13913 	mutex_lock(&swhash->hlist_mutex);  in perf_swevent_init_cpu()
13914 	if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) {  in perf_swevent_init_cpu()
13919 		rcu_assign_pointer(swhash->swevent_hlist, hlist);  in perf_swevent_init_cpu()
13921 	mutex_unlock(&swhash->hlist_mutex);  in perf_swevent_init_cpu()
13931 	raw_spin_lock(&ctx->lock);  in __perf_event_exit_context()
13933 	list_for_each_entry(event, &ctx->event_list, event_entry)  in __perf_event_exit_context()
13935 	raw_spin_unlock(&ctx->lock);  in __perf_event_exit_context()
13950 		target[scope] = -1;  in perf_event_clear_cpumask()
13963 		if (pmu->scope == PERF_PMU_SCOPE_NONE ||  in perf_event_clear_cpumask()
13964 		    WARN_ON_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE))  in perf_event_clear_cpumask()
13967 		if (target[pmu->scope] >= 0 && target[pmu->scope] < nr_cpu_ids)  in perf_event_clear_cpumask()
13968 			perf_pmu_migrate_context(pmu, cpu, target[pmu->scope]);  in perf_event_clear_cpumask()
13977 	// XXX simplify cpuctx->online  in perf_event_exit_cpu_context()
13985 	ctx = &cpuctx->ctx;  in perf_event_exit_cpu_context()
13987 	mutex_lock(&ctx->mutex);  in perf_event_exit_cpu_context()
13989 	cpuctx->online = 0;  in perf_event_exit_cpu_context()
13990 	mutex_unlock(&ctx->mutex);  in perf_event_exit_cpu_context()
14045 	ctx = &cpuctx->ctx;  in perf_event_init_cpu()
14047 	mutex_lock(&ctx->mutex);  in perf_event_init_cpu()
14048 	cpuctx->online = 1;  in perf_event_init_cpu()
14049 	mutex_unlock(&ctx->mutex);  in perf_event_init_cpu()
14090 	perf_pmu_register(&perf_cpu_clock, "cpu_clock", -1);  in perf_event_init()
14091 	perf_pmu_register(&perf_task_clock, "task_clock", -1);  in perf_event_init()
14115 	if (pmu_attr->event_str)  in perf_event_sysfs_show()
14116 		return sprintf(page, "%s\n", pmu_attr->event_str);  in perf_event_sysfs_show()
14134 		if (pmu->dev)  in perf_event_sysfs_init()
14138 		WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);  in perf_event_sysfs_init()
14158 		return ERR_PTR(-ENOMEM);  in perf_cgroup_css_alloc()
14160 	jc->info = alloc_percpu(struct perf_cgroup_info);  in perf_cgroup_css_alloc()
14161 	if (!jc->info) {  in perf_cgroup_css_alloc()
14163 		return ERR_PTR(-ENOMEM);  in perf_cgroup_css_alloc()
14166 	return &jc->css;  in perf_cgroup_css_alloc()
14173 	free_percpu(jc->info);  in perf_cgroup_css_free()
14179 	perf_event_cgroup(css->cgroup);  in perf_cgroup_css_online()