173 static int sf_buffer_available(struct cpu_hw_sf *cpuhw)  in sf_buffer_available()  argument
175 	return !!cpuhw->sfb.sdbt;  in sf_buffer_available()
405 static void deallocate_buffers(struct cpu_hw_sf *cpuhw)  in deallocate_buffers()  argument
407 	if (cpuhw->sfb.sdbt)  in deallocate_buffers()
408 		free_sampling_buffer(&cpuhw->sfb);  in deallocate_buffers()
411 static int allocate_buffers(struct cpu_hw_sf *cpuhw, struct hw_perf_event *hwc)  in allocate_buffers()  argument
444 	freq = sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc));  in allocate_buffers()
456 	if (sf_buffer_available(cpuhw))  in allocate_buffers()
459 	return alloc_sampling_buffer(&cpuhw->sfb,  in allocate_buffers()
460 				     sfb_pending_allocs(&cpuhw->sfb, hwc));  in allocate_buffers()
492 static void sfb_account_overflows(struct cpu_hw_sf *cpuhw,  in sfb_account_overflows()  argument
507 	ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb,  in sfb_account_overflows()
508 			     sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)));  in sfb_account_overflows()
511 	num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb);  in sfb_account_overflows()
560 	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);  in setup_pmc_cpu()  local
564 		memset(cpuhw, 0, sizeof(*cpuhw));  in setup_pmc_cpu()
565 		qsi(&cpuhw->qsi);  in setup_pmc_cpu()
566 		cpuhw->flags |= PMU_F_RESERVED;  in setup_pmc_cpu()
570 		cpuhw->flags &= ~PMU_F_RESERVED;  in setup_pmc_cpu()
572 		deallocate_buffers(cpuhw);  in setup_pmc_cpu()
750 	struct cpu_hw_sf *cpuhw;  in __hw_perf_event_init()  local
775 	cpuhw = NULL;  in __hw_perf_event_init()
782 		cpuhw = &per_cpu(cpu_hw_sf, event->cpu);  in __hw_perf_event_init()
783 		si = cpuhw->qsi;  in __hw_perf_event_init()
828 	if (cpuhw)  in __hw_perf_event_init()
830 		err = allocate_buffers(cpuhw, hwc);  in __hw_perf_event_init()
836 			cpuhw = &per_cpu(cpu_hw_sf, cpu);  in __hw_perf_event_init()
837 			err = allocate_buffers(cpuhw, hwc);  in __hw_perf_event_init()
909 	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumsf_pmu_enable()  local
913 	if (cpuhw->flags & PMU_F_ENABLED)  in cpumsf_pmu_enable()
916 	if (cpuhw->flags & PMU_F_ERR_MASK)  in cpumsf_pmu_enable()
930 	if (cpuhw->event) {  in cpumsf_pmu_enable()
931 		hwc = &cpuhw->event->hw;  in cpumsf_pmu_enable()
937 			sfb_account_overflows(cpuhw, hwc);  in cpumsf_pmu_enable()
938 			extend_sampling_buffer(&cpuhw->sfb, hwc);  in cpumsf_pmu_enable()
941 		cpuhw->lsctl.interval = SAMPL_RATE(hwc);  in cpumsf_pmu_enable()
945 	cpuhw->flags |= PMU_F_ENABLED;  in cpumsf_pmu_enable()
948 	err = lsctl(&cpuhw->lsctl);  in cpumsf_pmu_enable()
950 		cpuhw->flags &= ~PMU_F_ENABLED;  in cpumsf_pmu_enable()
961 	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumsf_pmu_disable()  local
966 	if (!(cpuhw->flags & PMU_F_ENABLED))  in cpumsf_pmu_disable()
969 	if (cpuhw->flags & PMU_F_ERR_MASK)  in cpumsf_pmu_disable()
973 	inactive = cpuhw->lsctl;  in cpumsf_pmu_disable()
992 		cpuhw->lsctl.tear = si.tear;  in cpumsf_pmu_disable()
993 		cpuhw->lsctl.dear = si.dear;  in cpumsf_pmu_disable()
996 	cpuhw->flags &= ~PMU_F_ENABLED;  in cpumsf_pmu_disable()
1357 			    struct cpu_hw_sf *cpuhw)  in aux_output_begin()  argument
1396 	cpuhw->lsctl.tear = virt_to_phys((void *)base) + offset * sizeof(unsigned long);  in aux_output_begin()
1397 	cpuhw->lsctl.dear = virt_to_phys((void *)aux->sdb_index[head]);  in aux_output_begin()
1515 static void hw_collect_aux(struct cpu_hw_sf *cpuhw)  in hw_collect_aux()  argument
1521 	struct perf_output_handle *handle = &cpuhw->handle;  in hw_collect_aux()
1537 		aux = perf_aux_output_begin(handle, cpuhw->event);  in hw_collect_aux()
1561 			perf_aux_output_end(&cpuhw->handle, size);  in hw_collect_aux()
1566 			perf_aux_output_end(&cpuhw->handle, size);  in hw_collect_aux()
1734 		struct cpu_hw_sf *cpuhw = &per_cpu(cpu_hw_sf, event->cpu);  in cpumsf_pmu_check_period()  local
1736 		si = cpuhw->qsi;  in cpumsf_pmu_check_period()
1755 	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumsf_pmu_start()  local
1761 	cpuhw->lsctl.cs = 1;  in cpumsf_pmu_start()
1763 		cpuhw->lsctl.cd = 1;  in cpumsf_pmu_start()
1772 	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumsf_pmu_stop()  local
1778 	cpuhw->lsctl.cs = 0;  in cpumsf_pmu_stop()
1779 	cpuhw->lsctl.cd = 0;  in cpumsf_pmu_stop()
1791 	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumsf_pmu_add()  local
1795 	if (cpuhw->flags & PMU_F_IN_USE)  in cpumsf_pmu_add()
1798 	if (!SAMPL_DIAG_MODE(&event->hw) && !cpuhw->sfb.sdbt)  in cpumsf_pmu_add()
1810 	cpuhw->lsctl.s = 0;  in cpumsf_pmu_add()
1811 	cpuhw->lsctl.h = 1;  in cpumsf_pmu_add()
1812 	cpuhw->lsctl.interval = SAMPL_RATE(&event->hw);  in cpumsf_pmu_add()
1814 		cpuhw->lsctl.tear = virt_to_phys(cpuhw->sfb.sdbt);  in cpumsf_pmu_add()
1815 		cpuhw->lsctl.dear = *(unsigned long *)cpuhw->sfb.sdbt;  in cpumsf_pmu_add()
1816 		TEAR_REG(&event->hw) = (unsigned long)cpuhw->sfb.sdbt;  in cpumsf_pmu_add()
1821 	if (WARN_ON_ONCE(cpuhw->lsctl.es == 1 || cpuhw->lsctl.ed == 1)) {  in cpumsf_pmu_add()
1826 		aux = perf_aux_output_begin(&cpuhw->handle, event);  in cpumsf_pmu_add()
1831 		err = aux_output_begin(&cpuhw->handle, aux, cpuhw);  in cpumsf_pmu_add()
1834 		cpuhw->lsctl.ed = 1;  in cpumsf_pmu_add()
1836 	cpuhw->lsctl.es = 1;  in cpumsf_pmu_add()
1839 	cpuhw->event = event;  in cpumsf_pmu_add()
1840 	cpuhw->flags |= PMU_F_IN_USE;  in cpumsf_pmu_add()
1852 	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumsf_pmu_del()  local
1857 	cpuhw->lsctl.es = 0;  in cpumsf_pmu_del()
1858 	cpuhw->lsctl.ed = 0;  in cpumsf_pmu_del()
1859 	cpuhw->flags &= ~PMU_F_IN_USE;  in cpumsf_pmu_del()
1860 	cpuhw->event = NULL;  in cpumsf_pmu_del()
1863 		aux_output_end(&cpuhw->handle);  in cpumsf_pmu_del()
1943 	struct cpu_hw_sf *cpuhw;  in cpumf_measurement_alert()  local
1948 	cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumf_measurement_alert()
1952 	if (!(cpuhw->flags & PMU_F_RESERVED))  in cpumf_measurement_alert()
1960 		if (cpuhw->flags & PMU_F_IN_USE)  in cpumf_measurement_alert()
1961 			if (SAMPL_DIAG_MODE(&cpuhw->event->hw))  in cpumf_measurement_alert()
1962 				hw_collect_aux(cpuhw);  in cpumf_measurement_alert()
1964 				hw_perf_event_update(cpuhw->event, 0);  in cpumf_measurement_alert()
1966 			WARN_ON_ONCE(!(cpuhw->flags & PMU_F_IN_USE));  in cpumf_measurement_alert()
1976 		qsi(&cpuhw->qsi);  in cpumf_measurement_alert()
1981 		cpuhw->flags |= PMU_F_ERR_LSDA;  in cpumf_measurement_alert()
1989 		cpuhw->flags |= PMU_F_ERR_IBE;  in cpumf_measurement_alert()