1 // SPDX-License-Identifier: GPL-2.0
3  * Performance event support - Processor Activity Instrumentation Extension
49 	struct pai_userdata *save;	/* Area to store non-zero counters */
52 	struct perf_event *event;	/* Perf event for sampling */  member
54 	struct list_head syswide_list;	/* List system-wide sampling events */
66 /* Free per CPU data when the last event is removed. */
77 /* On initialization of first event also allocate per CPU data dynamically.
90 			 * decrements refcnt when an event can not be  in paiext_root_alloc()
93 			return -ENOMEM;  in paiext_root_alloc()
109 /* Free all memory allocated for event counting/sampling setup */
112 	kfree(mp->mapptr->area);  in paiext_free()
113 	kfree(mp->mapptr->paiext_cb);  in paiext_free()
114 	kvfree(mp->mapptr->save);  in paiext_free()
115 	kfree(mp->mapptr);  in paiext_free()
116 	mp->mapptr = NULL;  in paiext_free()
119 /* Release the PMU if event is the last perf event */
120 static void paiext_event_destroy_cpu(struct perf_event *event, int cpu)  in paiext_event_destroy_cpu()  argument
123 	struct paiext_map *cpump = mp->mapptr;  in paiext_event_destroy_cpu()
126 	if (refcount_dec_and_test(&cpump->refcnt))	/* Last reference gone */  in paiext_event_destroy_cpu()
132 static void paiext_event_destroy(struct perf_event *event)  in paiext_event_destroy()  argument
136 	free_page(PAI_SAVE_AREA(event));  in paiext_event_destroy()
137 	if (event->cpu == -1) {  in paiext_event_destroy()
138 		struct cpumask *mask = PAI_CPU_MASK(event);  in paiext_event_destroy()
141 			paiext_event_destroy_cpu(event, cpu);  in paiext_event_destroy()
144 		paiext_event_destroy_cpu(event, event->cpu);  in paiext_event_destroy()
147 			    event->cpu);  in paiext_event_destroy()
153  * Only one instance of event pai_ext/NNPA_ALL/ for sampling is
154  * allowed and when this event is running, no counting event is allowed.
155  * Several counting events are allowed in parallel, but no sampling event
159  * When the event initialization functions fails, no other call back will
162  * Allocate the memory for the event.
164 static int paiext_alloc_cpu(struct perf_event *event, int cpu)  in paiext_alloc_cpu()  argument
176 	cpump = mp->mapptr;  in paiext_alloc_cpu()
178 		rc = -ENOMEM;  in paiext_alloc_cpu()
185 		 * - a 512 byte block and requires 512 byte boundary alignment.  in paiext_alloc_cpu()
186 		 * - a 1KB byte block and requires 1KB boundary alignment.  in paiext_alloc_cpu()
187 		 * Only the first counting event has to allocate the area.  in paiext_alloc_cpu()
193 		mp->mapptr = cpump;  in paiext_alloc_cpu()
194 		cpump->area = kzalloc(PAIE1_CTRBLOCK_SZ, GFP_KERNEL);  in paiext_alloc_cpu()
195 		cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL);  in paiext_alloc_cpu()
196 		cpump->save = kvmalloc_array(paiext_cnt + 1,  in paiext_alloc_cpu()
199 		if (!cpump->save || !cpump->area || !cpump->paiext_cb) {  in paiext_alloc_cpu()
203 		INIT_LIST_HEAD(&cpump->syswide_list);  in paiext_alloc_cpu()
204 		refcount_set(&cpump->refcnt, 1);  in paiext_alloc_cpu()
207 		refcount_inc(&cpump->refcnt);  in paiext_alloc_cpu()
212 		/* Error in allocation of event, decrement anchor. Since  in paiext_alloc_cpu()
213 		 * the event in not created, its destroy() function is never  in paiext_alloc_cpu()
220 	/* If rc is non-zero, no increment of counter/sampler was done. */  in paiext_alloc_cpu()
224 static int paiext_alloc(struct perf_event *event)  in paiext_alloc()  argument
227 	int cpu, rc = -ENOMEM;  in paiext_alloc()
234 		rc = paiext_alloc_cpu(event, cpu);  in paiext_alloc()
237 				paiext_event_destroy_cpu(event, cpu);  in paiext_alloc()
248 	 * for this event.  in paiext_alloc()
250 	PAI_CPU_MASK(event) = maskptr;  in paiext_alloc()
257  * the index within PAIE1_CB given the event number. Also validate event
260 static int paiext_event_valid(struct perf_event *event)  in paiext_event_valid()  argument
262 	u64 cfg = event->attr.config;  in paiext_event_valid()
266 		event->hw.config_base = offsetof(struct paiext_cb, acc);  in paiext_event_valid()
269 	return -EINVAL;  in paiext_event_valid()
272 /* Might be called on different CPU than the one the event is intended for. */
273 static int paiext_event_init(struct perf_event *event)  in paiext_event_init()  argument
275 	struct perf_event_attr *a = &event->attr;  in paiext_event_init()
278 	/* PMU pai_ext registered as PERF_TYPE_RAW, check event type */  in paiext_event_init()
279 	if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type)  in paiext_event_init()
280 		return -ENOENT;  in paiext_event_init()
281 	/* PAI extension event must be valid and in supported range */  in paiext_event_init()
282 	rc = paiext_event_valid(event);  in paiext_event_init()
285 	/* Allow only event NNPA_ALL for sampling. */  in paiext_event_init()
286 	if (a->sample_period && a->config != PAI_NNPA_BASE)  in paiext_event_init()
287 		return -EINVAL;  in paiext_event_init()
288 	/* Prohibit exclude_user event selection */  in paiext_event_init()
289 	if (a->exclude_user)  in paiext_event_init()
290 		return -EINVAL;  in paiext_event_init()
292 	if (a->sample_period) {  in paiext_event_init()
293 		PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL);  in paiext_event_init()
294 		if (!PAI_SAVE_AREA(event))  in paiext_event_init()
295 			return -ENOMEM;  in paiext_event_init()
298 	if (event->cpu >= 0)  in paiext_event_init()
299 		rc = paiext_alloc_cpu(event, event->cpu);  in paiext_event_init()
301 		rc = paiext_alloc(event);  in paiext_event_init()
303 		free_page(PAI_SAVE_AREA(event));  in paiext_event_init()
306 	event->destroy = paiext_event_destroy;  in paiext_event_init()
308 	if (a->sample_period) {  in paiext_event_init()
309 		a->sample_period = 1;  in paiext_event_init()
310 		a->freq = 0;  in paiext_event_init()
312 		event->attach_state |= PERF_ATTACH_SCHED_CB;  in paiext_event_init()
314 		a->sample_type |= PERF_SAMPLE_RAW;  in paiext_event_init()
316 		a->inherit = 0;  in paiext_event_init()
327 /* Read the counter values. Return value from location in buffer. For event
330 static u64 paiext_getdata(struct perf_event *event)  in paiext_getdata()  argument
333 	struct paiext_map *cpump = mp->mapptr;  in paiext_getdata()
337 	if (event->attr.config != PAI_NNPA_BASE)  in paiext_getdata()
338 		return paiext_getctr(cpump->area,  in paiext_getdata()
339 				     event->attr.config - PAI_NNPA_BASE);  in paiext_getdata()
342 		sum += paiext_getctr(cpump->area, i);  in paiext_getdata()
347 static u64 paiext_getall(struct perf_event *event)  in paiext_getall()  argument
349 	return paiext_getdata(event);  in paiext_getall()
352 static void paiext_read(struct perf_event *event)  in paiext_read()  argument
356 	prev = local64_read(&event->hw.prev_count);  in paiext_read()
357 	new = paiext_getall(event);  in paiext_read()
358 	local64_set(&event->hw.prev_count, new);  in paiext_read()
359 	delta = new - prev;  in paiext_read()
360 	local64_add(delta, &event->count);  in paiext_read()
363 static void paiext_start(struct perf_event *event, int flags)  in paiext_start()  argument
366 	struct paiext_map *cpump = mp->mapptr;  in paiext_start()
369 	if (!event->attr.sample_period) {	/* Counting */  in paiext_start()
370 		sum = paiext_getall(event);	/* Get current value */  in paiext_start()
371 		local64_set(&event->hw.prev_count, sum);  in paiext_start()
373 		memcpy((void *)PAI_SAVE_AREA(event), cpump->area,  in paiext_start()
375 		/* Enable context switch callback for system-wide sampling */  in paiext_start()
376 		if (!(event->attach_state & PERF_ATTACH_TASK)) {  in paiext_start()
377 			list_add_tail(PAI_SWLIST(event), &cpump->syswide_list);  in paiext_start()
378 			perf_sched_cb_inc(event->pmu);  in paiext_start()
380 			cpump->event = event;  in paiext_start()
385 static int paiext_add(struct perf_event *event, int flags)  in paiext_add()  argument
388 	struct paiext_map *cpump = mp->mapptr;  in paiext_add()
389 	struct paiext_cb *pcb = cpump->paiext_cb;  in paiext_add()
391 	if (++cpump->active_events == 1) {  in paiext_add()
392 		get_lowcore()->aicd = virt_to_phys(cpump->paiext_cb);  in paiext_add()
393 		pcb->acc = virt_to_phys(cpump->area) | 0x1;  in paiext_add()
398 		paiext_start(event, PERF_EF_RELOAD);  in paiext_add()
399 	event->hw.state = 0;  in paiext_add()
404 static void paiext_stop(struct perf_event *event, int flags)  in paiext_stop()  argument
407 	struct paiext_map *cpump = mp->mapptr;  in paiext_stop()
409 	if (!event->attr.sample_period) {	/* Counting */  in paiext_stop()
410 		paiext_read(event);  in paiext_stop()
412 		if (!(event->attach_state & PERF_ATTACH_TASK)) {  in paiext_stop()
413 			list_del(PAI_SWLIST(event));  in paiext_stop()
414 			perf_sched_cb_dec(event->pmu);  in paiext_stop()
416 			paiext_have_sample(event, cpump);  in paiext_stop()
417 			cpump->event = NULL;  in paiext_stop()
420 	event->hw.state = PERF_HES_STOPPED;  in paiext_stop()
423 static void paiext_del(struct perf_event *event, int flags)  in paiext_del()  argument
426 	struct paiext_map *cpump = mp->mapptr;  in paiext_del()
427 	struct paiext_cb *pcb = cpump->paiext_cb;  in paiext_del()
429 	paiext_stop(event, PERF_EF_UPDATE);  in paiext_del()
430 	if (--cpump->active_events == 0) {  in paiext_del()
433 		pcb->acc = 0;  in paiext_del()
434 		get_lowcore()->aicd = 0;  in paiext_del()
453 			val -= val_old;  in paiext_copy()
455 			val = (~0ULL - val_old) + val + 1;  in paiext_copy()
478  * returns and has deleted the event on that CPU.
481 			      struct perf_event *event)  in paiext_push_sample()  argument
492 	perf_sample_data_init(&data, 0, event->hw.last_period);  in paiext_push_sample()
493 	if (event->attr.sample_type & PERF_SAMPLE_TID) {  in paiext_push_sample()
497 	if (event->attr.sample_type & PERF_SAMPLE_TIME)  in paiext_push_sample()
498 		data.time = event->clock();  in paiext_push_sample()
499 	if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))  in paiext_push_sample()
500 		data.id = event->id;  in paiext_push_sample()
501 	if (event->attr.sample_type & PERF_SAMPLE_CPU)  in paiext_push_sample()
503 	if (event->attr.sample_type & PERF_SAMPLE_RAW) {  in paiext_push_sample()
505 		raw.frag.data = cpump->save;  in paiext_push_sample()
509 	overflow = perf_event_overflow(event, &data, &regs);  in paiext_push_sample()
510 	perf_event_update_userpage(event);  in paiext_push_sample()
511 	/* Save NNPA lowcore area after read in event */  in paiext_push_sample()
512 	memcpy((void *)PAI_SAVE_AREA(event), cpump->area,  in paiext_push_sample()
518 static void paiext_have_sample(struct perf_event *event,  in paiext_have_sample()  argument
523 	if (!event)  in paiext_have_sample()
525 	rawsize = paiext_copy(cpump->save, cpump->area,  in paiext_have_sample()
526 			      (unsigned long *)PAI_SAVE_AREA(event));  in paiext_have_sample()
528 		paiext_push_sample(rawsize, cpump, event);  in paiext_have_sample()
535 	struct paiext_map *cpump = mp->mapptr;  in paiext_have_samples()
536 	struct perf_event *event;  in paiext_have_samples()  local
538 	list_for_each_entry(event, &cpump->syswide_list, hw.tp_list)  in paiext_have_samples()
539 		paiext_have_sample(event, cpump);  in paiext_have_samples()
542 /* Called on schedule-in and schedule-out. No access to event structure,
543  * but for sampling only event NNPA_ALL is allowed.
547 	/* We started with a clean page on event installation. So read out  in paiext_sched_task()
572 PMU_FORMAT_ATTR(event, "config:0-63");
675 		return -ENOMEM;  in attr_event_init_one()
677 	sysfs_attr_init(&pa->attr.attr);  in attr_event_init_one()
678 	pa->id = PAI_NNPA_BASE + num;  in attr_event_init_one()
679 	pa->attr.attr.name = paiext_ctrnames[num];  in attr_event_init_one()
680 	pa->attr.attr.mode = 0444;  in attr_event_init_one()
681 	pa->attr.show = cpumf_events_sysfs_show;  in attr_event_init_one()
682 	pa->attr.store = NULL;  in attr_event_init_one()
683 	attrs[num] = &pa->attr.attr;  in attr_event_init_one()
687 /* Create PMU sysfs event attributes on the fly. */
695 		return -ENOMEM;  in attr_event_init()
711 	int rc = -ENOMEM;  in paiext_init()
733 		rc = -ENOMEM;  in paiext_init()
738 	rc = perf_pmu_register(&paiext, KMSG_COMPONENT, -1);  in paiext_init()