50 static bool is_idxd_event(struct idxd_pmu *idxd_pmu, struct perf_event *event)  in is_idxd_event()  argument
52 	return &idxd_pmu->pmu == event->pmu;  in is_idxd_event()
55 static int perfmon_collect_events(struct idxd_pmu *idxd_pmu,  in perfmon_collect_events()  argument
62 	max_count = idxd_pmu->n_counters;  in perfmon_collect_events()
63 	n = idxd_pmu->n_events;  in perfmon_collect_events()
68 	if (is_idxd_event(idxd_pmu, leader)) {  in perfmon_collect_events()
69 		idxd_pmu->event_list[n] = leader;  in perfmon_collect_events()
70 		idxd_pmu->event_list[n]->hw.idx = n;  in perfmon_collect_events()
78 		if (!is_idxd_event(idxd_pmu, event) ||  in perfmon_collect_events()
85 		idxd_pmu->event_list[n] = event;  in perfmon_collect_events()
86 		idxd_pmu->event_list[n]->hw.idx = n;  in perfmon_collect_events()
93 static void perfmon_assign_hw_event(struct idxd_pmu *idxd_pmu,  in perfmon_assign_hw_event()  argument
96 	struct idxd_device *idxd = idxd_pmu->idxd;  in perfmon_assign_hw_event()
104 static int perfmon_assign_event(struct idxd_pmu *idxd_pmu,  in perfmon_assign_event()  argument
110 		if (!test_and_set_bit(i, idxd_pmu->used_mask))  in perfmon_assign_event()
124 static int perfmon_validate_group(struct idxd_pmu *pmu,  in perfmon_validate_group()
128 	struct idxd_pmu *fake_pmu;  in perfmon_validate_group()
186 	if (event->pmu != &idxd->idxd_pmu->pmu)  in perfmon_pmu_event_init()
194 		ret = perfmon_validate_group(idxd->idxd_pmu, event);  in perfmon_pmu_event_init()
214 	int shift = 64 - idxd->idxd_pmu->counter_width;  in perfmon_pmu_event_update()
236 	n_counters = min(idxd->idxd_pmu->n_counters, OVERFLOW_SIZE);  in perfmon_counter_overflow()
254 			event = idxd->idxd_pmu->event_list[i];  in perfmon_counter_overflow()
317 	if (flt_wq && test_bit(FLT_WQ, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
319 	if (flt_tc && test_bit(FLT_TC, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
321 	if (flt_pg_sz && test_bit(FLT_PG_SZ, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
323 	if (flt_xfer_sz && test_bit(FLT_XFER_SZ, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
325 	if (flt_eng && test_bit(FLT_ENG, &idxd->idxd_pmu->supported_filters))  in perfmon_pmu_event_start()
351 	for (i = 0; i < idxd->idxd_pmu->n_events; i++) {  in perfmon_pmu_event_stop()
352 		if (event != idxd->idxd_pmu->event_list[i])  in perfmon_pmu_event_stop()
355 		for (++i; i < idxd->idxd_pmu->n_events; i++)  in perfmon_pmu_event_stop()
356 			idxd->idxd_pmu->event_list[i - 1] = idxd->idxd_pmu->event_list[i];  in perfmon_pmu_event_stop()
357 		--idxd->idxd_pmu->n_events;  in perfmon_pmu_event_stop()
369 	clear_bit(cntr, idxd->idxd_pmu->used_mask);  in perfmon_pmu_event_stop()
380 	struct idxd_pmu *idxd_pmu = idxd->idxd_pmu;  in perfmon_pmu_event_add()  local
384 	n = perfmon_collect_events(idxd_pmu, event, false);  in perfmon_pmu_event_add()
392 	idx = perfmon_assign_event(idxd_pmu, event);  in perfmon_pmu_event_add()
396 	perfmon_assign_hw_event(idxd_pmu, event, idx);  in perfmon_pmu_event_add()
401 	idxd_pmu->n_events = n;  in perfmon_pmu_event_add()
439 static void idxd_pmu_init(struct idxd_pmu *idxd_pmu)  in idxd_pmu_init()  argument
444 		if (!test_bit(i, &idxd_pmu->supported_filters))  in idxd_pmu_init()
448 	idxd_pmu->pmu.name		= idxd_pmu->name;  in idxd_pmu_init()
449 	idxd_pmu->pmu.attr_groups	= perfmon_attr_groups;  in idxd_pmu_init()
450 	idxd_pmu->pmu.task_ctx_nr	= perf_invalid_context;  in idxd_pmu_init()
451 	idxd_pmu->pmu.event_init	= perfmon_pmu_event_init;  in idxd_pmu_init()
452 	idxd_pmu->pmu.pmu_enable	= perfmon_pmu_enable;  in idxd_pmu_init()
453 	idxd_pmu->pmu.pmu_disable	= perfmon_pmu_disable;  in idxd_pmu_init()
454 	idxd_pmu->pmu.add		= perfmon_pmu_event_add;  in idxd_pmu_init()
455 	idxd_pmu->pmu.del		= perfmon_pmu_event_del;  in idxd_pmu_init()
456 	idxd_pmu->pmu.start		= perfmon_pmu_event_start;  in idxd_pmu_init()
457 	idxd_pmu->pmu.stop		= perfmon_pmu_event_stop;  in idxd_pmu_init()
458 	idxd_pmu->pmu.read		= perfmon_pmu_event_update;  in idxd_pmu_init()
459 	idxd_pmu->pmu.capabilities	= PERF_PMU_CAP_NO_EXCLUDE;  in idxd_pmu_init()
460 	idxd_pmu->pmu.scope		= PERF_PMU_SCOPE_SYS_WIDE;  in idxd_pmu_init()
461 	idxd_pmu->pmu.module		= THIS_MODULE;  in idxd_pmu_init()
466 	if (!idxd->idxd_pmu)  in perfmon_pmu_remove()
469 	perf_pmu_unregister(&idxd->idxd_pmu->pmu);  in perfmon_pmu_remove()
470 	kfree(idxd->idxd_pmu);  in perfmon_pmu_remove()
471 	idxd->idxd_pmu = NULL;  in perfmon_pmu_remove()
477 	struct idxd_pmu *idxd_pmu;  in perfmon_pmu_init()  local
487 	idxd_pmu = kzalloc(sizeof(*idxd_pmu), GFP_KERNEL);  in perfmon_pmu_init()
488 	if (!idxd_pmu)  in perfmon_pmu_init()
491 	idxd_pmu->idxd = idxd;  in perfmon_pmu_init()
492 	idxd->idxd_pmu = idxd_pmu;  in perfmon_pmu_init()
495 		rc = sprintf(idxd_pmu->name, "dsa%d", idxd->id);  in perfmon_pmu_init()
499 		rc = sprintf(idxd_pmu->name, "iax%d", idxd->id);  in perfmon_pmu_init()
536 	idxd_pmu->n_event_categories = perfcap.num_event_category;  in perfmon_pmu_init()
537 	idxd_pmu->supported_event_categories = perfcap.global_event_category;  in perfmon_pmu_init()
538 	idxd_pmu->per_counter_caps_supported = perfcap.cap_per_counter;  in perfmon_pmu_init()
541 	idxd_pmu->supported_filters = perfcap.filter;  in perfmon_pmu_init()
543 		idxd_pmu->n_filters = hweight8(perfcap.filter);  in perfmon_pmu_init()
546 	idxd_pmu->n_counters = perfcap.num_perf_counter;  in perfmon_pmu_init()
547 	idxd_pmu->counter_width = perfcap.counter_width;  in perfmon_pmu_init()
549 	idxd_pmu_init(idxd_pmu);  in perfmon_pmu_init()
551 	rc = perf_pmu_register(&idxd_pmu->pmu, idxd_pmu->name, -1);  in perfmon_pmu_init()
558 	kfree(idxd_pmu);  in perfmon_pmu_init()
559 	idxd->idxd_pmu = NULL;  in perfmon_pmu_init()