213 	struct cpu_cf_events *cpuhw;  in cpum_cf_free_cpu()  local
225 	cpuhw = p->cpucf;  in cpum_cf_free_cpu()
230 	if (!cpuhw)  in cpum_cf_free_cpu()
233 	if (refcount_dec_and_test(&cpuhw->refcnt)) {  in cpum_cf_free_cpu()
234 		kfree(cpuhw);  in cpum_cf_free_cpu()
245 	struct cpu_cf_events *cpuhw;  in cpum_cf_alloc_cpu()  local
254 	cpuhw = p->cpucf;  in cpum_cf_alloc_cpu()
256 	if (!cpuhw) {  in cpum_cf_alloc_cpu()
257 		cpuhw = kzalloc(sizeof(*cpuhw), GFP_KERNEL);  in cpum_cf_alloc_cpu()
258 		if (cpuhw) {  in cpum_cf_alloc_cpu()
259 			p->cpucf = cpuhw;  in cpum_cf_alloc_cpu()
260 			refcount_set(&cpuhw->refcnt, 1);  in cpum_cf_alloc_cpu()
265 		refcount_inc(&cpuhw->refcnt);  in cpum_cf_alloc_cpu()
558 static int cfdiag_diffctr(struct cpu_cf_events *cpuhw, unsigned long auth)  in cfdiag_diffctr()  argument
566 		ctrstart = (struct cf_ctrset_entry *)(cpuhw->start + offset);  in cfdiag_diffctr()
567 		ctrstop = (struct cf_ctrset_entry *)(cpuhw->stop + offset);  in cfdiag_diffctr()
590 	trailer_start = (struct cf_trailer_entry *)(cpuhw->start + offset);  in cfdiag_diffctr()
591 	trailer_stop = (struct cf_trailer_entry *)(cpuhw->stop + offset);  in cfdiag_diffctr()
677 	struct cpu_cf_events *cpuhw = this_cpu_cfhw();  in cpumf_pmu_enable()  local
680 	if (!cpuhw || (cpuhw->flags & PMU_F_ENABLED))  in cpumf_pmu_enable()
683 	err = lcctl(cpuhw->state | cpuhw->dev_state);  in cpumf_pmu_enable()
687 		cpuhw->flags |= PMU_F_ENABLED;  in cpumf_pmu_enable()
697 	struct cpu_cf_events *cpuhw = this_cpu_cfhw();  in cpumf_pmu_disable()  local
701 	if (!cpuhw || !(cpuhw->flags & PMU_F_ENABLED))  in cpumf_pmu_disable()
704 	inactive = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1);  in cpumf_pmu_disable()
705 	inactive |= cpuhw->dev_state;  in cpumf_pmu_disable()
710 		cpuhw->flags &= ~PMU_F_ENABLED;  in cpumf_pmu_disable()
927 	struct cpu_cf_events *cpuhw = this_cpu_cfhw();  in cpumf_pmu_start()  local
937 	ctr_set_enable(&cpuhw->state, hwc->config_base);  in cpumf_pmu_start()
938 	ctr_set_start(&cpuhw->state, hwc->config_base);  in cpumf_pmu_start()
946 		cpuhw->usedss = cfdiag_getctr(cpuhw->start,  in cpumf_pmu_start()
947 					      sizeof(cpuhw->start),  in cpumf_pmu_start()
956 			atomic_inc(&cpuhw->ctr_set[i]);  in cpumf_pmu_start()
967 			      struct cpu_cf_events *cpuhw)  in cfdiag_push_sample()  argument
982 		raw.frag.size = cpuhw->usedss;  in cfdiag_push_sample()
983 		raw.frag.data = cpuhw->stop;  in cfdiag_push_sample()
997 	struct cpu_cf_events *cpuhw = this_cpu_cfhw();  in cpumf_pmu_stop()  local
1009 			if (!atomic_dec_return(&cpuhw->ctr_set[i]))  in cpumf_pmu_stop()
1010 				ctr_set_stop(&cpuhw->state, cpumf_ctr_ctl[i]);  in cpumf_pmu_stop()
1018 			cpuhw->usedss = cfdiag_getctr(cpuhw->stop,  in cpumf_pmu_stop()
1019 						      sizeof(cpuhw->stop),  in cpumf_pmu_stop()
1022 			if (cfdiag_diffctr(cpuhw, event->hw.config_base))  in cpumf_pmu_stop()
1023 				cfdiag_push_sample(event, cpuhw);  in cpumf_pmu_stop()
1033 	struct cpu_cf_events *cpuhw = this_cpu_cfhw();  in cpumf_pmu_add()  local
1035 	ctr_set_enable(&cpuhw->state, event->hw.config_base);  in cpumf_pmu_add()
1046 	struct cpu_cf_events *cpuhw = this_cpu_cfhw();  in cpumf_pmu_del()  local
1060 		if (!atomic_read(&cpuhw->ctr_set[i]))  in cpumf_pmu_del()
1061 			ctr_set_disable(&cpuhw->state, cpumf_ctr_ctl[i]);  in cpumf_pmu_del()
1160 	struct cpu_cf_events *cpuhw;  in cpumf_measurement_alert()  local
1171 	cpuhw = this_cpu_cfhw();  in cpumf_measurement_alert()
1172 	if (!cpuhw)  in cpumf_measurement_alert()
1318 	struct cpu_cf_events *cpuhw = this_cpu_cfhw();  in cfset_ioctl_off()  local
1325 			if (!atomic_dec_return(&cpuhw->ctr_set[rc])) {  in cfset_ioctl_off()
1326 				ctr_set_disable(&cpuhw->dev_state,  in cfset_ioctl_off()
1328 				ctr_set_stop(&cpuhw->dev_state,  in cfset_ioctl_off()
1333 	rc = lcctl(cpuhw->dev_state | cpuhw->state);  in cfset_ioctl_off()
1336 		       cpuhw->state, S390_HWCTR_DEVICE, rc);  in cfset_ioctl_off()
1337 	if (!cpuhw->dev_state)  in cfset_ioctl_off()
1338 		cpuhw->flags &= ~PMU_F_IN_USE;  in cfset_ioctl_off()
1344 	struct cpu_cf_events *cpuhw = this_cpu_cfhw();  in cfset_ioctl_on()  local
1348 	cpuhw->flags |= PMU_F_IN_USE;  in cfset_ioctl_on()
1349 	ctr_set_enable(&cpuhw->dev_state, p->sets);  in cfset_ioctl_on()
1350 	ctr_set_start(&cpuhw->dev_state, p->sets);  in cfset_ioctl_on()
1353 			atomic_inc(&cpuhw->ctr_set[rc]);  in cfset_ioctl_on()
1354 	rc = lcctl(cpuhw->dev_state | cpuhw->state);	/* Start counter sets */  in cfset_ioctl_on()
1359 		       cpuhw->dev_state | cpuhw->state, S390_HWCTR_DEVICE, rc);  in cfset_ioctl_on()
1364 	struct cpu_cf_events *cpuhw = this_cpu_cfhw();  in cfset_release_cpu()  local
1367 	cpuhw->dev_state = 0;  in cfset_release_cpu()
1368 	rc = lcctl(cpuhw->state);	/* Keep perf_event_open counter sets */  in cfset_release_cpu()
1371 		       cpuhw->state, S390_HWCTR_DEVICE, rc);  in cfset_release_cpu()
1492 		struct cpu_cf_events *cpuhw = get_cpu_cfhw(cpu);  in cfset_all_copy()  local
1497 		rc |= put_user(cpuhw->sets, &ctrset_cpudata->no_sets);  in cfset_all_copy()
1498 		rc |= copy_to_user(ctrset_cpudata->data, cpuhw->data,  in cfset_all_copy()
1499 				   cpuhw->used);  in cfset_all_copy()
1504 		uptr += sizeof(struct s390_ctrset_cpudata) + cpuhw->used;  in cfset_all_copy()
1534 	struct cpu_cf_events *cpuhw = this_cpu_cfhw();  in cfset_cpu_read()  local
1540 	cpuhw->used = 0;  in cfset_cpu_read()
1541 	cpuhw->sets = 0;  in cfset_cpu_read()
1542 	memset(cpuhw->data, 0, sizeof(cpuhw->data));  in cfset_cpu_read()
1546 		struct s390_ctrset_setdata *sp = (void *)cpuhw->data +  in cfset_cpu_read()
1547 						 cpuhw->used;  in cfset_cpu_read()
1552 		space = sizeof(cpuhw->data) - cpuhw->used;  in cfset_cpu_read()
1555 			cpuhw->used += space;  in cfset_cpu_read()
1556 			cpuhw->sets += 1;  in cfset_cpu_read()