106 	void (*write_evttype)(struct xgene_pmu_dev *pmu_dev, int idx, u32 val);
107 	void (*write_agentmsk)(struct xgene_pmu_dev *pmu_dev, u32 val);
108 	void (*write_agent1msk)(struct xgene_pmu_dev *pmu_dev, u32 val);
109 	void (*enable_counter)(struct xgene_pmu_dev *pmu_dev, int idx);
110 	void (*disable_counter)(struct xgene_pmu_dev *pmu_dev, int idx);
111 	void (*enable_counter_int)(struct xgene_pmu_dev *pmu_dev, int idx);
112 	void (*disable_counter_int)(struct xgene_pmu_dev *pmu_dev, int idx);
113 	void (*reset_counters)(struct xgene_pmu_dev *pmu_dev);
114 	void (*start_counters)(struct xgene_pmu_dev *pmu_dev);
115 	void (*stop_counters)(struct xgene_pmu_dev *pmu_dev);
139 	struct xgene_pmu_dev *pmu_dev;  member
596 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(dev_get_drvdata(dev));  in cpumask_show()  local
598 	return cpumap_print_to_pagebuf(true, buf, &pmu_dev->parent->cpu);  in cpumask_show()
681 static int get_next_avail_cntr(struct xgene_pmu_dev *pmu_dev)  in get_next_avail_cntr()  argument
685 	cntr = find_first_zero_bit(pmu_dev->cntr_assign_mask,  in get_next_avail_cntr()
686 				pmu_dev->max_counters);  in get_next_avail_cntr()
687 	if (cntr == pmu_dev->max_counters)  in get_next_avail_cntr()
689 	set_bit(cntr, pmu_dev->cntr_assign_mask);  in get_next_avail_cntr()
694 static void clear_avail_cntr(struct xgene_pmu_dev *pmu_dev, int cntr)  in clear_avail_cntr()  argument
696 	clear_bit(cntr, pmu_dev->cntr_assign_mask);  in clear_avail_cntr()
720 static inline u64 xgene_pmu_read_counter32(struct xgene_pmu_dev *pmu_dev,  in xgene_pmu_read_counter32()  argument
723 	return readl(pmu_dev->inf->csr + PMU_PMEVCNTR0 + (4 * idx));  in xgene_pmu_read_counter32()
726 static inline u64 xgene_pmu_read_counter64(struct xgene_pmu_dev *pmu_dev,  in xgene_pmu_read_counter64()  argument
738 		hi = xgene_pmu_read_counter32(pmu_dev, 2 * idx + 1);  in xgene_pmu_read_counter64()
739 		lo = xgene_pmu_read_counter32(pmu_dev, 2 * idx);  in xgene_pmu_read_counter64()
740 	} while (hi != xgene_pmu_read_counter32(pmu_dev, 2 * idx + 1));  in xgene_pmu_read_counter64()
746 xgene_pmu_write_counter32(struct xgene_pmu_dev *pmu_dev, int idx, u64 val)  in xgene_pmu_write_counter32()  argument
748 	writel(val, pmu_dev->inf->csr + PMU_PMEVCNTR0 + (4 * idx));  in xgene_pmu_write_counter32()
752 xgene_pmu_write_counter64(struct xgene_pmu_dev *pmu_dev, int idx, u64 val)  in xgene_pmu_write_counter64()  argument
760 	xgene_pmu_write_counter32(pmu_dev, 2 * idx, cnt_lo);  in xgene_pmu_write_counter64()
761 	xgene_pmu_write_counter32(pmu_dev, 2 * idx + 1, cnt_hi);  in xgene_pmu_write_counter64()
765 xgene_pmu_write_evttype(struct xgene_pmu_dev *pmu_dev, int idx, u32 val)  in xgene_pmu_write_evttype()  argument
767 	writel(val, pmu_dev->inf->csr + PMU_PMEVTYPER0 + (4 * idx));  in xgene_pmu_write_evttype()
771 xgene_pmu_write_agentmsk(struct xgene_pmu_dev *pmu_dev, u32 val)  in xgene_pmu_write_agentmsk()  argument
773 	writel(val, pmu_dev->inf->csr + PMU_PMAMR0);  in xgene_pmu_write_agentmsk()
777 xgene_pmu_v3_write_agentmsk(struct xgene_pmu_dev *pmu_dev, u32 val) { }  in xgene_pmu_v3_write_agentmsk()  argument
780 xgene_pmu_write_agent1msk(struct xgene_pmu_dev *pmu_dev, u32 val)  in xgene_pmu_write_agent1msk()  argument
782 	writel(val, pmu_dev->inf->csr + PMU_PMAMR1);  in xgene_pmu_write_agent1msk()
786 xgene_pmu_v3_write_agent1msk(struct xgene_pmu_dev *pmu_dev, u32 val) { }  in xgene_pmu_v3_write_agent1msk()  argument
789 xgene_pmu_enable_counter(struct xgene_pmu_dev *pmu_dev, int idx)  in xgene_pmu_enable_counter()  argument
793 	val = readl(pmu_dev->inf->csr + PMU_PMCNTENSET);  in xgene_pmu_enable_counter()
795 	writel(val, pmu_dev->inf->csr + PMU_PMCNTENSET);  in xgene_pmu_enable_counter()
799 xgene_pmu_disable_counter(struct xgene_pmu_dev *pmu_dev, int idx)  in xgene_pmu_disable_counter()  argument
803 	val = readl(pmu_dev->inf->csr + PMU_PMCNTENCLR);  in xgene_pmu_disable_counter()
805 	writel(val, pmu_dev->inf->csr + PMU_PMCNTENCLR);  in xgene_pmu_disable_counter()
809 xgene_pmu_enable_counter_int(struct xgene_pmu_dev *pmu_dev, int idx)  in xgene_pmu_enable_counter_int()  argument
813 	val = readl(pmu_dev->inf->csr + PMU_PMINTENSET);  in xgene_pmu_enable_counter_int()
815 	writel(val, pmu_dev->inf->csr + PMU_PMINTENSET);  in xgene_pmu_enable_counter_int()
819 xgene_pmu_disable_counter_int(struct xgene_pmu_dev *pmu_dev, int idx)  in xgene_pmu_disable_counter_int()  argument
823 	val = readl(pmu_dev->inf->csr + PMU_PMINTENCLR);  in xgene_pmu_disable_counter_int()
825 	writel(val, pmu_dev->inf->csr + PMU_PMINTENCLR);  in xgene_pmu_disable_counter_int()
828 static inline void xgene_pmu_reset_counters(struct xgene_pmu_dev *pmu_dev)  in xgene_pmu_reset_counters()  argument
832 	val = readl(pmu_dev->inf->csr + PMU_PMCR);  in xgene_pmu_reset_counters()
834 	writel(val, pmu_dev->inf->csr + PMU_PMCR);  in xgene_pmu_reset_counters()
837 static inline void xgene_pmu_start_counters(struct xgene_pmu_dev *pmu_dev)  in xgene_pmu_start_counters()  argument
841 	val = readl(pmu_dev->inf->csr + PMU_PMCR);  in xgene_pmu_start_counters()
843 	writel(val, pmu_dev->inf->csr + PMU_PMCR);  in xgene_pmu_start_counters()
846 static inline void xgene_pmu_stop_counters(struct xgene_pmu_dev *pmu_dev)  in xgene_pmu_stop_counters()  argument
850 	val = readl(pmu_dev->inf->csr + PMU_PMCR);  in xgene_pmu_stop_counters()
852 	writel(val, pmu_dev->inf->csr + PMU_PMCR);  in xgene_pmu_stop_counters()
857 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(pmu);  in xgene_perf_pmu_enable()  local
858 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_pmu_enable()
859 	bool enabled = !bitmap_empty(pmu_dev->cntr_assign_mask,  in xgene_perf_pmu_enable()
860 			pmu_dev->max_counters);  in xgene_perf_pmu_enable()
865 	xgene_pmu->ops->start_counters(pmu_dev);  in xgene_perf_pmu_enable()
870 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(pmu);  in xgene_perf_pmu_disable()  local
871 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_pmu_disable()
873 	xgene_pmu->ops->stop_counters(pmu_dev);  in xgene_perf_pmu_disable()
878 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_event_init()  local
905 	event->cpu = cpumask_first(&pmu_dev->parent->cpu);  in xgene_perf_event_init()
935 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_enable_event()  local
936 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_enable_event()
938 	xgene_pmu->ops->write_evttype(pmu_dev, GET_CNTR(event),  in xgene_perf_enable_event()
940 	xgene_pmu->ops->write_agentmsk(pmu_dev, ~((u32)GET_AGENTID(event)));  in xgene_perf_enable_event()
941 	if (pmu_dev->inf->type == PMU_TYPE_IOB)  in xgene_perf_enable_event()
942 		xgene_pmu->ops->write_agent1msk(pmu_dev,  in xgene_perf_enable_event()
945 	xgene_pmu->ops->enable_counter(pmu_dev, GET_CNTR(event));  in xgene_perf_enable_event()
946 	xgene_pmu->ops->enable_counter_int(pmu_dev, GET_CNTR(event));  in xgene_perf_enable_event()
951 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_disable_event()  local
952 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_disable_event()
954 	xgene_pmu->ops->disable_counter(pmu_dev, GET_CNTR(event));  in xgene_perf_disable_event()
955 	xgene_pmu->ops->disable_counter_int(pmu_dev, GET_CNTR(event));  in xgene_perf_disable_event()
960 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_event_set_period()  local
961 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_event_set_period()
973 	xgene_pmu->ops->write_counter(pmu_dev, hw->idx, val);  in xgene_perf_event_set_period()
978 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_event_update()  local
979 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_event_update()
985 	new_raw_count = xgene_pmu->ops->read_counter(pmu_dev, GET_CNTR(event));  in xgene_perf_event_update()
991 	delta = (new_raw_count - prev_raw_count) & pmu_dev->max_period;  in xgene_perf_event_update()
1003 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_start()  local
1004 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in xgene_perf_start()
1018 		xgene_pmu->ops->write_counter(pmu_dev, GET_CNTR(event),  in xgene_perf_start()
1046 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_add()  local
1052 	hw->idx = get_next_avail_cntr(pmu_dev);  in xgene_perf_add()
1057 	pmu_dev->pmu_counter_event[hw->idx] = event;  in xgene_perf_add()
1067 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);  in xgene_perf_del()  local
1073 	clear_avail_cntr(pmu_dev, GET_CNTR(event));  in xgene_perf_del()
1076 	pmu_dev->pmu_counter_event[hw->idx] = NULL;  in xgene_perf_del()
1079 static int xgene_init_perf(struct xgene_pmu_dev *pmu_dev, char *name)  in xgene_init_perf()  argument
1083 	if (pmu_dev->parent->version == PCP_PMU_V3)  in xgene_init_perf()
1084 		pmu_dev->max_period = PMU_V3_CNT_MAX_PERIOD;  in xgene_init_perf()
1086 		pmu_dev->max_period = PMU_CNT_MAX_PERIOD;  in xgene_init_perf()
1088 	xgene_pmu = pmu_dev->parent;  in xgene_init_perf()
1090 		pmu_dev->max_counters = 1;  in xgene_init_perf()
1092 		pmu_dev->max_counters = PMU_MAX_COUNTERS;  in xgene_init_perf()
1095 	pmu_dev->pmu = (struct pmu) {  in xgene_init_perf()
1096 		.parent		= pmu_dev->parent->dev,  in xgene_init_perf()
1097 		.attr_groups	= pmu_dev->attr_groups,  in xgene_init_perf()
1111 	xgene_pmu->ops->stop_counters(pmu_dev);  in xgene_init_perf()
1112 	xgene_pmu->ops->reset_counters(pmu_dev);  in xgene_init_perf()
1114 	return perf_pmu_register(&pmu_dev->pmu, name, -1);  in xgene_init_perf()
1128 	ctx->pmu_dev = pmu;  in xgene_pmu_dev_add()
1179 static void _xgene_pmu_isr(int irq, struct xgene_pmu_dev *pmu_dev)  in _xgene_pmu_isr()  argument
1181 	struct xgene_pmu *xgene_pmu = pmu_dev->parent;  in _xgene_pmu_isr()
1182 	void __iomem *csr = pmu_dev->inf->csr;  in _xgene_pmu_isr()
1186 	xgene_pmu->ops->stop_counters(pmu_dev);  in _xgene_pmu_isr()
1205 		struct perf_event *event = pmu_dev->pmu_counter_event[idx];  in _xgene_pmu_isr()
1216 	xgene_pmu->ops->start_counters(pmu_dev);  in _xgene_pmu_isr()
1243 			_xgene_pmu_isr(irq, ctx->pmu_dev);  in xgene_pmu_isr()
1248 			_xgene_pmu_isr(irq, ctx->pmu_dev);  in xgene_pmu_isr()
1253 			_xgene_pmu_isr(irq, ctx->pmu_dev);  in xgene_pmu_isr()
1258 			_xgene_pmu_isr(irq, ctx->pmu_dev);  in xgene_pmu_isr()
1808 		perf_pmu_migrate_context(&ctx->pmu_dev->pmu, cpu, target);  in xgene_pmu_offline_cpu()
1811 		perf_pmu_migrate_context(&ctx->pmu_dev->pmu, cpu, target);  in xgene_pmu_offline_cpu()
1814 		perf_pmu_migrate_context(&ctx->pmu_dev->pmu, cpu, target);  in xgene_pmu_offline_cpu()
1817 		perf_pmu_migrate_context(&ctx->pmu_dev->pmu, cpu, target);  in xgene_pmu_offline_cpu()
1928 		perf_pmu_unregister(&ctx->pmu_dev->pmu);  in xgene_pmu_dev_cleanup()