124 #define cd_to_ni(cd) container_of((cd), struct arm_ni, cds[(cd)->id])  argument
127 #define cd_for_each_unit(cd, u) \  argument
128 	for (struct arm_ni_unit *u = cd->units; u < cd->units + cd->num_units; u++)
158 	struct arm_ni_cd *cd = pmu_to_cd(dev_get_drvdata(dev));  in arm_ni_event_attr_is_visible()  local
163 	cd_for_each_unit(cd, unit) {  in arm_ni_event_attr_is_visible()
221 	struct arm_ni_cd *cd = pmu_to_cd(dev_get_drvdata(dev));  in arm_ni_cpumask_show()  local
223 	return cpumap_print_to_pagebuf(true, buf, cpumask_of(cd->cpu));  in arm_ni_cpumask_show()
310 	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);  in arm_ni_event_init()  local
318 	event->cpu = cd->cpu;  in arm_ni_event_init()
322 	cd_for_each_unit(cd, unit) {  in arm_ni_event_init()
332 static u64 arm_ni_read_ccnt(struct arm_ni_cd *cd)  in arm_ni_read_ccnt()  argument
337 	u_new = readl_relaxed(cd->pmu_base + NI_PMCCNTR_U);  in arm_ni_read_ccnt()
340 		l = readl_relaxed(cd->pmu_base + NI_PMCCNTR_L);  in arm_ni_read_ccnt()
341 		u_new = readl_relaxed(cd->pmu_base + NI_PMCCNTR_U);  in arm_ni_read_ccnt()
350 	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);  in arm_ni_event_read()  local
358 			count = arm_ni_read_ccnt(cd);  in arm_ni_event_read()
360 			count = readl_relaxed(cd->pmu_base + NI_PMEVCNTR(hw->idx));  in arm_ni_event_read()
371 	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);  in arm_ni_event_start()  local
373 	writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENSET);  in arm_ni_event_start()
378 	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);  in arm_ni_event_stop()  local
380 	writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENCLR);  in arm_ni_event_stop()
385 static void arm_ni_init_ccnt(struct arm_ni_cd *cd)  in arm_ni_init_ccnt()  argument
387 	local64_set(&cd->ccnt->hw.prev_count, S64_MIN);  in arm_ni_init_ccnt()
388 	lo_hi_writeq_relaxed(S64_MIN, cd->pmu_base + NI_PMCCNTR_L);  in arm_ni_init_ccnt()
391 static void arm_ni_init_evcnt(struct arm_ni_cd *cd, int idx)  in arm_ni_init_evcnt()  argument
393 	local64_set(&cd->evcnt[idx]->hw.prev_count, S32_MIN);  in arm_ni_init_evcnt()
394 	writel_relaxed(S32_MIN, cd->pmu_base + NI_PMEVCNTR(idx));  in arm_ni_init_evcnt()
399 	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);  in arm_ni_event_add()  local
406 		if (cd->ccnt)  in arm_ni_event_add()
409 		cd->ccnt = event;  in arm_ni_event_add()
410 		arm_ni_init_ccnt(cd);  in arm_ni_event_add()
413 		while (cd->evcnt[hw->idx]) {  in arm_ni_event_add()
417 		cd->evcnt[hw->idx] = event;  in arm_ni_event_add()
420 		arm_ni_init_evcnt(cd, hw->idx);  in arm_ni_event_add()
425 		writel_relaxed(reg, cd->pmu_base + NI_PMEVTYPER(hw->idx));  in arm_ni_event_add()
434 	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);  in arm_ni_event_del()  local
440 		cd->ccnt = NULL;  in arm_ni_event_del()
442 		cd->evcnt[hw->idx] = NULL;  in arm_ni_event_del()
447 	struct arm_ni_cd *cd = dev_id;  in arm_ni_handle_irq()  local
449 	u32 reg = readl_relaxed(cd->pmu_base + NI_PMOVSCLR);  in arm_ni_handle_irq()
453 		if (!(WARN_ON(!cd->ccnt))) {  in arm_ni_handle_irq()
454 			arm_ni_event_read(cd->ccnt);  in arm_ni_handle_irq()
455 			arm_ni_init_ccnt(cd);  in arm_ni_handle_irq()
462 		if (!(WARN_ON(!cd->evcnt[i]))) {  in arm_ni_handle_irq()
463 			arm_ni_event_read(cd->evcnt[i]);  in arm_ni_handle_irq()
464 			arm_ni_init_evcnt(cd, i);  in arm_ni_handle_irq()
467 	writel_relaxed(reg, cd->pmu_base + NI_PMOVSCLR);  in arm_ni_handle_irq()
473 	struct arm_ni_cd *cd = ni->cds + node->id;  in arm_ni_init_cd()  local
477 	cd->id = node->id;  in arm_ni_init_cd()
478 	cd->num_units = node->num_components;  in arm_ni_init_cd()
479 	cd->units = devm_kcalloc(ni->dev, cd->num_units, sizeof(*(cd->units)), GFP_KERNEL);  in arm_ni_init_cd()
480 	if (!cd->units)  in arm_ni_init_cd()
483 	for (int i = 0; i < cd->num_units; i++) {  in arm_ni_init_cd()
486 		struct arm_ni_unit *unit = cd->units + i;  in arm_ni_init_cd()
496 				dev_info(ni->dev, "No access to PMU %d\n", cd->id);  in arm_ni_init_cd()
497 				devm_kfree(ni->dev, cd->units);  in arm_ni_init_cd()
501 			cd->pmu_base = unit_base;  in arm_ni_init_cd()
525 	res_start += cd->pmu_base - ni->base;  in arm_ni_init_cd()
532 		       cd->pmu_base + NI_PMCR);  in arm_ni_init_cd()
533 	writel_relaxed(U32_MAX, cd->pmu_base + NI_PMCNTENCLR);  in arm_ni_init_cd()
534 	writel_relaxed(U32_MAX, cd->pmu_base + NI_PMOVSCLR);  in arm_ni_init_cd()
535 	writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENSET);  in arm_ni_init_cd()
537 	cd->irq = platform_get_irq(to_platform_device(ni->dev), cd->id);  in arm_ni_init_cd()
538 	if (cd->irq < 0)  in arm_ni_init_cd()
539 		return cd->irq;  in arm_ni_init_cd()
541 	err = devm_request_irq(ni->dev, cd->irq, arm_ni_handle_irq,  in arm_ni_init_cd()
543 			       dev_name(ni->dev), cd);  in arm_ni_init_cd()
547 	cd->cpu = cpumask_local_spread(0, dev_to_node(ni->dev));  in arm_ni_init_cd()
548 	cd->pmu = (struct pmu) {  in arm_ni_init_cd()
564 	name = devm_kasprintf(ni->dev, GFP_KERNEL, "arm_ni_%d_cd_%d", ni->id, cd->id);  in arm_ni_init_cd()
568 	err = cpuhp_state_add_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node);  in arm_ni_init_cd()
572 	err = perf_pmu_register(&cd->pmu, name, -1);  in arm_ni_init_cd()
574 		cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node);  in arm_ni_init_cd()
591 	struct arm_ni_node cfg, vd, pd, cd;  in arm_ni_probe()  local
658 				arm_ni_probe_domain(base + reg, &cd);  in arm_ni_probe()
659 				ret = arm_ni_init_cd(ni, &cd, res->start);  in arm_ni_probe()
674 		struct arm_ni_cd *cd = ni->cds + i;  in arm_ni_remove()  local
676 		if (!cd->pmu_base)  in arm_ni_remove()
679 		writel_relaxed(0, cd->pmu_base + NI_PMCR);  in arm_ni_remove()
680 		writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENCLR);  in arm_ni_remove()
681 		perf_pmu_unregister(&cd->pmu);  in arm_ni_remove()
682 		cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node);  in arm_ni_remove()
712 static void arm_ni_pmu_migrate(struct arm_ni_cd *cd, unsigned int cpu)  in arm_ni_pmu_migrate()  argument
714 	perf_pmu_migrate_context(&cd->pmu, cd->cpu, cpu);  in arm_ni_pmu_migrate()
715 	irq_set_affinity(cd->irq, cpumask_of(cpu));  in arm_ni_pmu_migrate()
716 	cd->cpu = cpu;  in arm_ni_pmu_migrate()
721 	struct arm_ni_cd *cd;  in arm_ni_pmu_online_cpu()  local
724 	cd = hlist_entry_safe(cpuhp_node, struct arm_ni_cd, cpuhp_node);  in arm_ni_pmu_online_cpu()
725 	node = dev_to_node(cd_to_ni(cd)->dev);  in arm_ni_pmu_online_cpu()
726 	if (cpu_to_node(cd->cpu) != node && cpu_to_node(cpu) == node)  in arm_ni_pmu_online_cpu()
727 		arm_ni_pmu_migrate(cd, cpu);  in arm_ni_pmu_online_cpu()
733 	struct arm_ni_cd *cd;  in arm_ni_pmu_offline_cpu()  local
737 	cd = hlist_entry_safe(cpuhp_node, struct arm_ni_cd, cpuhp_node);  in arm_ni_pmu_offline_cpu()
738 	if (cpu != cd->cpu)  in arm_ni_pmu_offline_cpu()
741 	node = dev_to_node(cd_to_ni(cd)->dev);  in arm_ni_pmu_offline_cpu()
747 		arm_ni_pmu_migrate(cd, target);  in arm_ni_pmu_offline_cpu()