393 static inline int amd_has_nb(struct cpu_hw_events *cpuc)  in amd_has_nb()  argument
395 	struct amd_nb *nb = cpuc->amd_nb;  in amd_has_nb()
421 static void __amd_put_nb_event_constraints(struct cpu_hw_events *cpuc,  in __amd_put_nb_event_constraints()  argument
424 	struct amd_nb *nb = cpuc->amd_nb;  in __amd_put_nb_event_constraints()
480 __amd_get_nb_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,  in __amd_get_nb_event_constraints()  argument
484 	struct amd_nb *nb = cpuc->amd_nb;  in __amd_get_nb_event_constraints()
491 	if (cpuc->is_fake)  in __amd_get_nb_event_constraints()
578 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);  in amd_pmu_cpu_prepare()  local
580 	cpuc->lbr_sel = kzalloc_node(sizeof(struct er_account), GFP_KERNEL,  in amd_pmu_cpu_prepare()
582 	if (!cpuc->lbr_sel)  in amd_pmu_cpu_prepare()
585 	WARN_ON_ONCE(cpuc->amd_nb);  in amd_pmu_cpu_prepare()
590 	cpuc->amd_nb = amd_alloc_nb(cpu);  in amd_pmu_cpu_prepare()
591 	if (cpuc->amd_nb)  in amd_pmu_cpu_prepare()
594 	kfree(cpuc->lbr_sel);  in amd_pmu_cpu_prepare()
595 	cpuc->lbr_sel = NULL;  in amd_pmu_cpu_prepare()
602 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);  in amd_pmu_cpu_starting()  local
603 	void **onln = &cpuc->kfree_on_online[X86_PERF_KFREE_SHARED];  in amd_pmu_cpu_starting()
607 	cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;  in amd_pmu_cpu_starting()
622 			*onln = cpuc->amd_nb;  in amd_pmu_cpu_starting()
623 			cpuc->amd_nb = nb;  in amd_pmu_cpu_starting()
628 	cpuc->amd_nb->nb_id = nb_id;  in amd_pmu_cpu_starting()
629 	cpuc->amd_nb->refcnt++;  in amd_pmu_cpu_starting()
723 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in amd_pmu_check_overflow()  local
741 		if (!test_bit(idx, cpuc->active_mask))  in amd_pmu_check_overflow()
755 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in amd_pmu_enable_all()  local
762 		if (!test_bit(idx, cpuc->active_mask))  in amd_pmu_enable_all()
765 		amd_pmu_enable_event(cpuc->events[idx]);  in amd_pmu_enable_all()
885 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in amd_pmu_handle_irq()  local
893 	pmu_enabled = cpuc->enabled;  in amd_pmu_handle_irq()
894 	cpuc->enabled = 0;  in amd_pmu_handle_irq()
899 	if (cpuc->lbr_users)  in amd_pmu_handle_irq()
905 	cpuc->enabled = pmu_enabled;  in amd_pmu_handle_irq()
920 	struct cpu_hw_events *cpuc;  in amd_pmu_v2_snapshot_branch_stack()  local
931 	cpuc = this_cpu_ptr(&cpu_hw_events);  in amd_pmu_v2_snapshot_branch_stack()
935 	memcpy(entries, cpuc->lbr_entries, sizeof(struct perf_branch_entry) * cnt);  in amd_pmu_v2_snapshot_branch_stack()
945 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in amd_pmu_v2_handle_irq()  local
957 	pmu_enabled = cpuc->enabled;  in amd_pmu_v2_handle_irq()
958 	cpuc->enabled = 0;  in amd_pmu_v2_handle_irq()
984 		if (!test_bit(idx, cpuc->active_mask))  in amd_pmu_v2_handle_irq()
987 		event = cpuc->events[idx];  in amd_pmu_v2_handle_irq()
1004 			perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);  in amd_pmu_v2_handle_irq()
1027 	cpuc->enabled = pmu_enabled;  in amd_pmu_v2_handle_irq()
1037 amd_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in amd_get_event_constraints()  argument
1043 	if (!(amd_has_nb(cpuc) && amd_is_nb_event(&event->hw)))  in amd_get_event_constraints()
1046 	return __amd_get_nb_event_constraints(cpuc, event, NULL);  in amd_get_event_constraints()
1049 static void amd_put_event_constraints(struct cpu_hw_events *cpuc,  in amd_put_event_constraints()  argument
1052 	if (amd_has_nb(cpuc) && amd_is_nb_event(&event->hw))  in amd_put_event_constraints()
1053 		__amd_put_nb_event_constraints(cpuc, event);  in amd_put_event_constraints()
1146 amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, int idx,  in amd_get_event_constraints_f15h()  argument
1221 amd_get_event_constraints_f17h(struct cpu_hw_events *cpuc, int idx,  in amd_get_event_constraints_f17h()  argument
1232 static void amd_put_event_constraints_f17h(struct cpu_hw_events *cpuc,  in amd_put_event_constraints_f17h()  argument
1238 		--cpuc->n_pair;  in amd_put_event_constraints_f17h()
1262 amd_get_event_constraints_f19h(struct cpu_hw_events *cpuc, int idx,  in amd_get_event_constraints_f19h()  argument
1560 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in amd_pmu_enable_virt()  local
1562 	cpuc->perf_ctr_virt_mask = 0;  in amd_pmu_enable_virt()
1571 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in amd_pmu_disable_virt()  local
1579 	cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;  in amd_pmu_disable_virt()