Lines Matching +full:- +full:chn +full:- +full:disabled
1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2016-2020 Arm Limited
3 // CMN-600 Coherent Mesh Network PMU driver
11 #include <linux/io-64-nonatomic-lo-hi.h>
36 #define CMN_MAX_DTMS (CMN_MAX_XPS + (CMN_MAX_DIMENSION - 1) * 4)
128 /* DTC counters are paired in 64-bit registers on a 16-byte stride. Yuck */
130 #define CMN_DT_PMEVCNT(dtc, n) ((dtc)->pmu_base + _CMN_DT_CNT_REG(n))
131 #define CMN_DT_PMCCNTR(dtc) ((dtc)->pmu_base + 0x40)
133 #define CMN_DT_PMEVCNTSR(dtc, n) ((dtc)->pmu_base + 0x50 + _CMN_DT_CNT_REG(n))
134 #define CMN_DT_PMCCNTRSR(dtc) ((dtc)->pmu_base + 0x90)
136 #define CMN_DT_PMCR(dtc) ((dtc)->pmu_base + 0x100)
141 #define CMN_DT_PMOVSR(dtc) ((dtc)->pmu_base + 0x118)
142 #define CMN_DT_PMOVSR_CLR(dtc) ((dtc)->pmu_base + 0x120)
144 #define CMN_DT_PMSSR(dtc) ((dtc)->pmu_base + 0x128)
147 #define CMN_DT_PMSRR(dtc) ((dtc)->pmu_base + 0x130)
158 /* Similarly for the 40-bit cycle counter */
169 #define CMN_EVENT_TYPE(event) FIELD_GET(CMN_CONFIG_TYPE, (event)->attr.config)
170 #define CMN_EVENT_EVENTID(event) FIELD_GET(CMN_CONFIG_EVENTID, (event)->attr.config)
171 #define CMN_EVENT_OCCUPID(event) FIELD_GET(CMN_CONFIG_OCCUPID, (event)->attr.config)
172 #define CMN_EVENT_BYNODEID(event) FIELD_GET(CMN_CONFIG_BYNODEID, (event)->attr.config)
173 #define CMN_EVENT_NODEID(event) FIELD_GET(CMN_CONFIG_NODEID, (event)->attr.config)
183 #define CMN_EVENT_WP_COMBINE(event) FIELD_GET(CMN_CONFIG_WP_COMBINE, (event)->attr.config)
184 #define CMN_EVENT_WP_DEV_SEL(event) FIELD_GET(CMN_CONFIG_WP_DEV_SEL, (event)->attr.config)
185 #define CMN_EVENT_WP_CHN_SEL(event) FIELD_GET(CMN_CONFIG_WP_CHN_SEL, (event)->attr.config)
186 #define CMN_EVENT_WP_GRP(event) FIELD_GET(CMN_CONFIG_WP_GRP, (event)->attr.config)
187 #define CMN_EVENT_WP_EXCLUSIVE(event) FIELD_GET(CMN_CONFIG_WP_EXCLUSIVE, (event)->attr.config)
188 #define CMN_EVENT_WP_VAL(event) FIELD_GET(CMN_CONFIG1_WP_VAL, (event)->attr.config1)
189 #define CMN_EVENT_WP_MASK(event) FIELD_GET(CMN_CONFIG2_WP_MASK, (event)->attr.config2)
191 /* Made-up event IDs for watchpoint direction */
204 CMN_ANY = -1,
205 NOT_CMN600 = -2,
218 /* CMN-600 r0px shouldn't exist in silicon, thankfully */
273 SEL_NONE = -1,
292 /* DN/HN-F/CXHA */
374 return fls((cmn->mesh_x - 1) | (cmn->mesh_y - 1)); in arm_cmn_xyidbits()
381 nid.dev = dn->id & ((1U << dn->deviceid_bits) - 1); in arm_cmn_nid()
382 nid.port = (dn->id >> dn->deviceid_bits) & ((1U << dn->portid_bits) - 1); in arm_cmn_nid()
389 int id = dn->id >> (dn->portid_bits + dn->deviceid_bits); in arm_cmn_node_to_xp()
392 int y = id & ((1U << bits) - 1); in arm_cmn_node_to_xp()
394 return cmn->xps + cmn->mesh_x * y + x; in arm_cmn_node_to_xp()
401 for (dn = cmn->dns; dn->type; dn++) in arm_cmn_node()
402 if (dn->type == type) in arm_cmn_node()
409 switch (cmn->part) { in arm_cmn_model()
427 if (cmn->part == PART_CMN_S3) { in arm_cmn_pmu_offset()
428 if (dn->type == CMN_TYPE_XP) in arm_cmn_pmu_offset()
438 int offset = CMN_MXP__CONNECT_INFO(port) - arm_cmn_pmu_offset(cmn, xp); in arm_cmn_device_connect_info()
441 if (cmn->part == PART_CMN600 || cmn->part == PART_CMN650) in arm_cmn_device_connect_info()
444 * CI-700 may have extra ports, but still has the in arm_cmn_device_connect_info()
447 if (cmn->part == PART_CI700) in arm_cmn_device_connect_info()
451 return readl_relaxed(xp->pmu_base + offset); in arm_cmn_device_connect_info()
461 case 0x01: return " RN-I |"; in arm_cmn_device_type()
462 case 0x02: return " RN-D |"; in arm_cmn_device_type()
463 case 0x04: return " RN-F_B |"; in arm_cmn_device_type()
464 case 0x05: return "RN-F_B_E|"; in arm_cmn_device_type()
465 case 0x06: return " RN-F_A |"; in arm_cmn_device_type()
466 case 0x07: return "RN-F_A_E|"; in arm_cmn_device_type()
467 case 0x08: return " HN-T |"; in arm_cmn_device_type()
468 case 0x09: return " HN-I |"; in arm_cmn_device_type()
469 case 0x0a: return " HN-D |"; in arm_cmn_device_type()
470 case 0x0b: return " HN-P |"; in arm_cmn_device_type()
471 case 0x0c: return " SN-F |"; in arm_cmn_device_type()
473 case 0x0e: return " HN-F |"; in arm_cmn_device_type()
474 case 0x0f: return " SN-F_E |"; in arm_cmn_device_type()
475 case 0x10: return " SN-F_D |"; in arm_cmn_device_type()
479 case 0x14: return " RN-F_D |"; in arm_cmn_device_type()
480 case 0x15: return "RN-F_D_E|"; in arm_cmn_device_type()
481 case 0x16: return " RN-F_C |"; in arm_cmn_device_type()
482 case 0x17: return "RN-F_C_E|"; in arm_cmn_device_type()
483 case 0x18: return " RN-F_E |"; in arm_cmn_device_type()
484 case 0x19: return "RN-F_E_E|"; in arm_cmn_device_type()
485 case 0x1a: return " HN-S |"; in arm_cmn_device_type()
488 case 0x1d: return " HN-V |"; in arm_cmn_device_type()
490 case 0x20: return " RN-F_F |"; in arm_cmn_device_type()
491 case 0x21: return "RN-F_F_E|"; in arm_cmn_device_type()
492 case 0x22: return " SN-F_F |"; in arm_cmn_device_type()
499 struct arm_cmn *cmn = s->private; in arm_cmn_show_logid()
501 u16 id = xp->id | d | (p << xp->deviceid_bits); in arm_cmn_show_logid()
503 for (dn = cmn->dns; dn->type; dn++) { in arm_cmn_show_logid()
504 int pad = dn->logid < 10; in arm_cmn_show_logid()
506 if (dn->type == CMN_TYPE_XP) in arm_cmn_show_logid()
509 if (dn->type < CMN_TYPE_HNI) in arm_cmn_show_logid()
512 if (dn->id != id) in arm_cmn_show_logid()
515 seq_printf(s, " %*c#%-*d |", pad + 1, ' ', 3 - pad, dn->logid); in arm_cmn_show_logid()
523 struct arm_cmn *cmn = s->private; in arm_cmn_map_show()
524 int x, y, p, pmax = fls(cmn->ports_used); in arm_cmn_map_show()
527 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
528 seq_printf(s, " %-2d ", x); in arm_cmn_map_show()
530 y = cmn->mesh_y; in arm_cmn_map_show()
531 while (y--) { in arm_cmn_map_show()
532 int xp_base = cmn->mesh_x * y; in arm_cmn_map_show()
533 struct arm_cmn_node *xp = cmn->xps + xp_base; in arm_cmn_map_show()
536 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
537 seq_puts(s, "--------+"); in arm_cmn_map_show()
539 seq_printf(s, "\n%-2d |", y); in arm_cmn_map_show()
540 for (x = 0; x < cmn->mesh_x; x++) { in arm_cmn_map_show()
543 seq_printf(s, " XP #%-3d|", xp_base + x); in arm_cmn_map_show()
547 for (x = 0; x < cmn->mesh_x; x++) { in arm_cmn_map_show()
556 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
561 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
564 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
567 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
570 seq_puts(s, "\n-----+"); in arm_cmn_map_show()
572 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
573 seq_puts(s, "--------+"); in arm_cmn_map_show()
584 name = devm_kasprintf(cmn->dev, GFP_KERNEL, "map_%d", id); in arm_cmn_debugfs_init()
588 cmn->debug = debugfs_create_file(name, 0444, arm_cmn_debugfs, cmn, &arm_cmn_map_fops); in arm_cmn_debugfs_init()
613 for (i = 0, dn = hw->dn; i < hw->num_dns; i++, dn++)
617 for (int i = 0, idx; i < CMN_MAX_DTCS; i++) if ((idx = hw->dtc_idx[i]) >= 0)
621 return (struct arm_cmn_hw_event *)&event->hw; in to_cmn_hw()
679 if (eattr->type == CMN_TYPE_DTC) in arm_cmn_event_show()
680 return sysfs_emit(buf, "type=0x%x\n", eattr->type); in arm_cmn_event_show()
682 if (eattr->type == CMN_TYPE_WP) in arm_cmn_event_show()
685 eattr->type, eattr->eventid); in arm_cmn_event_show()
687 if (eattr->fsel > SEL_NONE) in arm_cmn_event_show()
689 eattr->type, eattr->eventid, eattr->occupid); in arm_cmn_event_show()
691 return sysfs_emit(buf, "type=0x%x,eventid=0x%x\n", eattr->type, in arm_cmn_event_show()
692 eattr->eventid); in arm_cmn_event_show()
707 if (!(eattr->model & arm_cmn_model(cmn))) in arm_cmn_event_attr_is_visible()
710 type = eattr->type; in arm_cmn_event_attr_is_visible()
711 eventid = eattr->eventid; in arm_cmn_event_attr_is_visible()
715 return attr->mode; in arm_cmn_event_attr_is_visible()
722 if ((intf & 4) && !(cmn->ports_used & BIT(intf & 3))) in arm_cmn_event_attr_is_visible()
725 if (chan == 4 && cmn->part == PART_CMN600) in arm_cmn_event_attr_is_visible()
728 if ((chan == 5 && cmn->rsp_vc_num < 2) || in arm_cmn_event_attr_is_visible()
729 (chan == 6 && cmn->dat_vc_num < 2) || in arm_cmn_event_attr_is_visible()
730 (chan == 7 && cmn->snp_vc_num < 2) || in arm_cmn_event_attr_is_visible()
731 (chan == 8 && cmn->req_vc_num < 2)) in arm_cmn_event_attr_is_visible()
735 /* Revision-specific differences */ in arm_cmn_event_attr_is_visible()
736 if (cmn->part == PART_CMN600) { in arm_cmn_event_attr_is_visible()
737 if (cmn->rev < REV_CMN600_R1P3) { in arm_cmn_event_attr_is_visible()
741 if (cmn->rev < REV_CMN600_R1P2) { in arm_cmn_event_attr_is_visible()
747 } else if (cmn->part == PART_CMN650) { in arm_cmn_event_attr_is_visible()
748 if (cmn->rev < REV_CMN650_R2P0 || cmn->rev == REV_CMN650_R1P2) { in arm_cmn_event_attr_is_visible()
756 } else if (cmn->part == PART_CMN700) { in arm_cmn_event_attr_is_visible()
757 if (cmn->rev < REV_CMN700_R2P0) { in arm_cmn_event_attr_is_visible()
765 if (cmn->rev < REV_CMN700_R1P0) { in arm_cmn_event_attr_is_visible()
774 return attr->mode; in arm_cmn_event_attr_is_visible()
899 * DVM node events conflict with HN-I events in the equivalent PMU
900 * slot, but our lazy short-cut of using the DTM counter index for
998 * HN-P events squat on top of the HN-I similarly to DVM events, except
1021 /* We treat watchpoints as a special made-up class of XP events */
1284 if (!fmt->config) in arm_cmn_format_show()
1285 return sysfs_emit(buf, "config:%*pbl\n", 64, &fmt->field); in arm_cmn_format_show()
1287 return sysfs_emit(buf, "config%d:%*pbl\n", fmt->config, 64, &fmt->field); in arm_cmn_format_show()
1327 return cpumap_print_to_pagebuf(true, buf, cpumask_of(cmn->cpu)); in arm_cmn_cpumask_show()
1338 return sysfs_emit(buf, "%03x%02x\n", cmn->part, cmn->rev); in arm_cmn_identifier_show()
1366 if (dtm->wp_event[wp_idx] >= 0) in arm_cmn_find_free_wp_idx()
1367 if (dtm->wp_event[++wp_idx] >= 0) in arm_cmn_find_free_wp_idx()
1368 return -ENOSPC; in arm_cmn_find_free_wp_idx()
1377 return CMN_EVENT_EVENTID(event) + arm_cmn_get_wp_idx(hw->wp_idx, pos); in arm_cmn_get_assigned_wp_idx()
1387 dtm->wp_event[wp_idx] = hw->dtc_idx[dtc]; in arm_cmn_claim_wp_idx()
1388 arm_cmn_set_wp_idx(hw->wp_idx, pos, wp_idx - CMN_EVENT_EVENTID(event)); in arm_cmn_claim_wp_idx()
1395 u32 chn = CMN_EVENT_WP_CHN_SEL(event); in arm_cmn_wp_config() local
1399 bool is_cmn600 = to_cmn(event->pmu)->part == PART_CMN600; in arm_cmn_wp_config()
1401 /* CMN-600 supports only primary and secondary matching groups */ in arm_cmn_wp_config()
1406 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_CHN_SEL, chn) | in arm_cmn_wp_config()
1422 if (!cmn->state) in arm_cmn_set_state()
1423 writel_relaxed(0, CMN_DT_PMCR(&cmn->dtc[0])); in arm_cmn_set_state()
1424 cmn->state |= state; in arm_cmn_set_state()
1429 cmn->state &= ~state; in arm_cmn_clear_state()
1430 if (!cmn->state) in arm_cmn_clear_state()
1432 CMN_DT_PMCR(&cmn->dtc[0])); in arm_cmn_clear_state()
1455 if (dtm != &cmn->dtms[dn->dtm]) { in arm_cmn_read_dtm()
1456 dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset; in arm_cmn_read_dtm()
1457 reg = readq_relaxed(dtm->base + offset); in arm_cmn_read_dtm()
1459 dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); in arm_cmn_read_dtm()
1471 return (val - CMN_CC_INIT) & ((CMN_CC_INIT << 1) - 1); in arm_cmn_read_cc()
1480 return val - CMN_COUNTER_INIT; in arm_cmn_read_counter()
1485 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_init_counter()
1490 writel_relaxed(CMN_COUNTER_INIT, CMN_DT_PMEVCNT(&cmn->dtc[i], idx)); in arm_cmn_init_counter()
1491 cmn->dtc[i].counters[idx] = event; in arm_cmn_init_counter()
1495 local64_set(&event->hw.prev_count, count); in arm_cmn_init_counter()
1500 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_read()
1506 delta = arm_cmn_read_cc(cmn->dtc + hw->dtc_idx[0]); in arm_cmn_event_read()
1507 local64_add(delta, &event->count); in arm_cmn_event_read()
1511 prev = local64_xchg(&event->hw.prev_count, new); in arm_cmn_event_read()
1513 delta = new - prev; in arm_cmn_event_read()
1517 new = arm_cmn_read_counter(cmn->dtc + i, idx); in arm_cmn_event_read()
1521 local64_add(delta, &event->count); in arm_cmn_event_read()
1532 if (!dn->occupid[fsel].count) { in arm_cmn_set_event_sel_hi()
1533 dn->occupid[fsel].val = occupid; in arm_cmn_set_event_sel_hi()
1535 dn->occupid[SEL_CBUSY_SNTHROTTLE_SEL].val) | in arm_cmn_set_event_sel_hi()
1537 dn->occupid[SEL_SN_HOME_SEL].val) | in arm_cmn_set_event_sel_hi()
1539 dn->occupid[SEL_HBT_LBT_SEL].val) | in arm_cmn_set_event_sel_hi()
1541 dn->occupid[SEL_CLASS_OCCUP_ID].val) | in arm_cmn_set_event_sel_hi()
1543 dn->occupid[SEL_OCCUP1ID].val); in arm_cmn_set_event_sel_hi()
1544 writel_relaxed(reg >> 32, dn->pmu_base + CMN_PMU_EVENT_SEL + 4); in arm_cmn_set_event_sel_hi()
1545 } else if (dn->occupid[fsel].val != occupid) { in arm_cmn_set_event_sel_hi()
1546 return -EBUSY; in arm_cmn_set_event_sel_hi()
1548 dn->occupid[fsel].count++; in arm_cmn_set_event_sel_hi()
1556 dn->event_w[dtm_idx] = eventid; in arm_cmn_set_event_sel_lo()
1557 writeq_relaxed(le64_to_cpu(dn->event_sel_w), dn->pmu_base + CMN_PMU_EVENT_SEL); in arm_cmn_set_event_sel_lo()
1559 dn->event[dtm_idx] = eventid; in arm_cmn_set_event_sel_lo()
1560 writel_relaxed(le32_to_cpu(dn->event_sel), dn->pmu_base + CMN_PMU_EVENT_SEL); in arm_cmn_set_event_sel_lo()
1566 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_start()
1573 struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0]; in arm_cmn_event_start()
1576 dtc->base + CMN_DT_DTC_CTL); in arm_cmn_event_start()
1578 dtc->cc_active = true; in arm_cmn_event_start()
1584 void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset); in arm_cmn_event_start()
1591 int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); in arm_cmn_event_start()
1594 hw->wide_sel); in arm_cmn_event_start()
1600 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_stop()
1607 struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0]; in arm_cmn_event_stop()
1609 dtc->cc_active = false; in arm_cmn_event_stop()
1610 writel_relaxed(CMN_DT_DTC_CTL_DT_EN, dtc->base + CMN_DT_DTC_CTL); in arm_cmn_event_stop()
1613 void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset); in arm_cmn_event_stop()
1620 int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); in arm_cmn_event_stop()
1622 arm_cmn_set_event_sel_lo(dn, dtm_idx, 0, hw->wide_sel); in arm_cmn_event_stop()
1642 if (val->wp[dtm][wp_idx]) in arm_cmn_val_find_free_wp_config()
1643 if (val->wp[dtm][++wp_idx]) in arm_cmn_val_find_free_wp_config()
1644 return -ENOSPC; in arm_cmn_val_find_free_wp_config()
1662 val->cycles = true; in arm_cmn_val_add_event()
1667 val->dtc_count[dtc]++; in arm_cmn_val_add_event()
1670 int wp_idx, dtm = dn->dtm, sel = hw->filter_sel; in arm_cmn_val_add_event()
1672 val->dtm_count[dtm]++; in arm_cmn_val_add_event()
1675 val->occupid[dtm][sel] = CMN_EVENT_OCCUPID(event) + 1; in arm_cmn_val_add_event()
1681 val->wp[dtm][wp_idx] = 1; in arm_cmn_val_add_event()
1682 val->wp_combine[dtm][wp_idx >> 1] += !!CMN_EVENT_WP_COMBINE(event); in arm_cmn_val_add_event()
1690 struct perf_event *sibling, *leader = event->group_leader; in arm_cmn_validate_group()
1693 int i, ret = -EINVAL; in arm_cmn_validate_group()
1698 if (event->pmu != leader->pmu && !is_software_event(leader)) in arm_cmn_validate_group()
1699 return -EINVAL; in arm_cmn_validate_group()
1703 return -ENOMEM; in arm_cmn_validate_group()
1712 ret = val->cycles ? -EINVAL : 0; in arm_cmn_validate_group()
1717 if (val->dtc_count[i] == CMN_DT_NUM_COUNTERS) in arm_cmn_validate_group()
1721 int wp_idx, dtm = dn->dtm, sel = hw->filter_sel; in arm_cmn_validate_group()
1723 if (val->dtm_count[dtm] == CMN_DTM_NUM_COUNTERS) in arm_cmn_validate_group()
1726 if (sel > SEL_NONE && val->occupid[dtm][sel] && in arm_cmn_validate_group()
1727 val->occupid[dtm][sel] != CMN_EVENT_OCCUPID(event) + 1) in arm_cmn_validate_group()
1738 val->wp_combine[dtm][wp_idx >> 1] != !!CMN_EVENT_WP_COMBINE(event)) in arm_cmn_validate_group()
1755 for (int i = 0; i < ARRAY_SIZE(arm_cmn_event_attrs) - 1; i++) { in arm_cmn_filter_sel()
1757 if (e->model & model && e->type == type && e->eventid == eventid) in arm_cmn_filter_sel()
1758 return e->fsel; in arm_cmn_filter_sel()
1766 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_init()
1773 if (event->attr.type != event->pmu->type) in arm_cmn_event_init()
1774 return -ENOENT; in arm_cmn_event_init()
1776 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) in arm_cmn_event_init()
1777 return -EINVAL; in arm_cmn_event_init()
1779 event->cpu = cmn->cpu; in arm_cmn_event_init()
1780 if (event->cpu < 0) in arm_cmn_event_init()
1781 return -EINVAL; in arm_cmn_event_init()
1794 return -EINVAL; in arm_cmn_event_init()
1796 if (cmn->multi_dtm) in arm_cmn_event_init()
1797 hw->dtm_offset = CMN_EVENT_WP_DEV_SEL(event) / 2; in arm_cmn_event_init()
1799 (cmn->part == PART_CMN700 || cmn->part == PART_CMN_S3)) { in arm_cmn_event_init()
1800 hw->wide_sel = true; in arm_cmn_event_init()
1804 hw->filter_sel = arm_cmn_filter_sel(cmn, type, eventid); in arm_cmn_event_init()
1809 hw->dn = arm_cmn_node(cmn, type); in arm_cmn_event_init()
1810 if (!hw->dn) in arm_cmn_event_init()
1811 return -EINVAL; in arm_cmn_event_init()
1813 memset(hw->dtc_idx, -1, sizeof(hw->dtc_idx)); in arm_cmn_event_init()
1814 for (dn = hw->dn; dn->type == type; dn++) { in arm_cmn_event_init()
1815 if (bynodeid && dn->id != nodeid) { in arm_cmn_event_init()
1816 hw->dn++; in arm_cmn_event_init()
1819 hw->num_dns++; in arm_cmn_event_init()
1820 if (dn->dtc < 0) in arm_cmn_event_init()
1821 memset(hw->dtc_idx, 0, cmn->num_dtcs); in arm_cmn_event_init()
1823 hw->dtc_idx[dn->dtc] = 0; in arm_cmn_event_init()
1829 if (!hw->num_dns) { in arm_cmn_event_init()
1830 dev_dbg(cmn->dev, "invalid node 0x%x type 0x%x\n", nodeid, type); in arm_cmn_event_init()
1831 return -EINVAL; in arm_cmn_event_init()
1843 while (i--) { in arm_cmn_event_clear()
1844 struct arm_cmn_dtm *dtm = &cmn->dtms[hw->dn[i].dtm] + hw->dtm_offset; in arm_cmn_event_clear()
1845 unsigned int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); in arm_cmn_event_clear()
1850 dtm->wp_event[wp_idx] = -1; in arm_cmn_event_clear()
1853 if (hw->filter_sel > SEL_NONE) in arm_cmn_event_clear()
1854 hw->dn[i].occupid[hw->filter_sel].count--; in arm_cmn_event_clear()
1856 dtm->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx); in arm_cmn_event_clear()
1857 writel_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG); in arm_cmn_event_clear()
1859 memset(hw->dtm_idx, 0, sizeof(hw->dtm_idx)); in arm_cmn_event_clear()
1860 memset(hw->wp_idx, 0, sizeof(hw->wp_idx)); in arm_cmn_event_clear()
1863 cmn->dtc[j].counters[idx] = NULL; in arm_cmn_event_clear()
1868 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_add()
1875 while (cmn->dtc[i].cycles) in arm_cmn_event_add()
1876 if (++i == cmn->num_dtcs) in arm_cmn_event_add()
1877 return -ENOSPC; in arm_cmn_event_add()
1879 cmn->dtc[i].cycles = event; in arm_cmn_event_add()
1880 hw->dtc_idx[0] = i; in arm_cmn_event_add()
1889 if (cmn->part == PART_CMN600 && j > 0) { in arm_cmn_event_add()
1890 idx = hw->dtc_idx[0]; in arm_cmn_event_add()
1893 while (cmn->dtc[j].counters[idx]) in arm_cmn_event_add()
1895 return -ENOSPC; in arm_cmn_event_add()
1897 hw->dtc_idx[j] = idx; in arm_cmn_event_add()
1902 struct arm_cmn_dtm *dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset; in arm_cmn_event_add()
1903 unsigned int dtm_idx, shift, d = max_t(int, dn->dtc, 0); in arm_cmn_event_add()
1907 while (dtm->pmu_config_low & CMN__PMEVCNT_PAIRED(dtm_idx)) in arm_cmn_event_add()
1923 tmp = dtm->wp_event[wp_idx ^ 1]; in arm_cmn_event_add()
1925 CMN_EVENT_WP_COMBINE(cmn->dtc[d].counters[tmp])) in arm_cmn_event_add()
1931 writel_relaxed(cfg, dtm->base + CMN_DTM_WPn_CONFIG(wp_idx)); in arm_cmn_event_add()
1935 if (cmn->multi_dtm) in arm_cmn_event_add()
1941 if (arm_cmn_set_event_sel_hi(dn, hw->filter_sel, CMN_EVENT_OCCUPID(event))) in arm_cmn_event_add()
1945 arm_cmn_set_index(hw->dtm_idx, i, dtm_idx); in arm_cmn_event_add()
1947 dtm->input_sel[dtm_idx] = input_sel; in arm_cmn_event_add()
1949 dtm->pmu_config_low &= ~(CMN__PMEVCNT0_GLOBAL_NUM << shift); in arm_cmn_event_add()
1950 dtm->pmu_config_low |= FIELD_PREP(CMN__PMEVCNT0_GLOBAL_NUM, hw->dtc_idx[d]) << shift; in arm_cmn_event_add()
1951 dtm->pmu_config_low |= CMN__PMEVCNT_PAIRED(dtm_idx); in arm_cmn_event_add()
1952 reg = (u64)le32_to_cpu(dtm->pmu_config_high) << 32 | dtm->pmu_config_low; in arm_cmn_event_add()
1953 writeq_relaxed(reg, dtm->base + CMN_DTM_PMU_CONFIG); in arm_cmn_event_add()
1966 return -ENOSPC; in arm_cmn_event_add()
1971 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_del()
1978 cmn->dtc[hw->dtc_idx[0]].cycles = NULL; in arm_cmn_event_del()
1980 arm_cmn_event_clear(cmn, event, hw->num_dns); in arm_cmn_event_del()
2009 perf_pmu_migrate_context(&cmn->pmu, cmn->cpu, cpu); in arm_cmn_migrate()
2010 for (i = 0; i < cmn->num_dtcs; i++) in arm_cmn_migrate()
2011 irq_set_affinity(cmn->dtc[i].irq, cpumask_of(cpu)); in arm_cmn_migrate()
2012 cmn->cpu = cpu; in arm_cmn_migrate()
2021 node = dev_to_node(cmn->dev); in arm_cmn_pmu_online_cpu()
2022 if (cpu_to_node(cmn->cpu) != node && cpu_to_node(cpu) == node) in arm_cmn_pmu_online_cpu()
2034 if (cpu != cmn->cpu) in arm_cmn_pmu_offline_cpu()
2037 node = dev_to_node(cmn->dev); in arm_cmn_pmu_offline_cpu()
2062 if (WARN_ON(!dtc->counters[i])) in arm_cmn_handle_irq()
2065 local64_add(delta, &dtc->counters[i]->count); in arm_cmn_handle_irq()
2071 if (dtc->cc_active && !WARN_ON(!dtc->cycles)) { in arm_cmn_handle_irq()
2073 local64_add(delta, &dtc->cycles->count); in arm_cmn_handle_irq()
2079 if (!dtc->irq_friend) in arm_cmn_handle_irq()
2081 dtc += dtc->irq_friend; in arm_cmn_handle_irq()
2090 for (i = 0; i < cmn->num_dtcs; i++) { in arm_cmn_init_irqs()
2091 irq = cmn->dtc[i].irq; in arm_cmn_init_irqs()
2092 for (j = i; j--; ) { in arm_cmn_init_irqs()
2093 if (cmn->dtc[j].irq == irq) { in arm_cmn_init_irqs()
2094 cmn->dtc[j].irq_friend = i - j; in arm_cmn_init_irqs()
2098 err = devm_request_irq(cmn->dev, irq, arm_cmn_handle_irq, in arm_cmn_init_irqs()
2100 dev_name(cmn->dev), &cmn->dtc[i]); in arm_cmn_init_irqs()
2104 err = irq_set_affinity(irq, cpumask_of(cmn->cpu)); in arm_cmn_init_irqs()
2117 dtm->base = xp->pmu_base + CMN_DTM_OFFSET(idx); in arm_cmn_init_dtm()
2118 dtm->pmu_config_low = CMN_DTM_PMU_CONFIG_PMU_EN; in arm_cmn_init_dtm()
2119 writeq_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG); in arm_cmn_init_dtm()
2121 dtm->wp_event[i] = -1; in arm_cmn_init_dtm()
2122 writeq_relaxed(0, dtm->base + CMN_DTM_WPn_MASK(i)); in arm_cmn_init_dtm()
2123 writeq_relaxed(~0ULL, dtm->base + CMN_DTM_WPn_VAL(i)); in arm_cmn_init_dtm()
2129 struct arm_cmn_dtc *dtc = cmn->dtc + idx; in arm_cmn_init_dtc()
2131 dtc->pmu_base = dn->pmu_base; in arm_cmn_init_dtc()
2132 dtc->base = dtc->pmu_base - arm_cmn_pmu_offset(cmn, dn); in arm_cmn_init_dtc()
2133 dtc->irq = platform_get_irq(to_platform_device(cmn->dev), idx); in arm_cmn_init_dtc()
2134 if (dtc->irq < 0) in arm_cmn_init_dtc()
2135 return dtc->irq; in arm_cmn_init_dtc()
2137 writel_relaxed(CMN_DT_DTC_CTL_DT_EN, dtc->base + CMN_DT_DTC_CTL); in arm_cmn_init_dtc()
2150 cmp = dna->type - dnb->type; in arm_cmn_node_cmp()
2152 cmp = dna->logid - dnb->logid; in arm_cmn_node_cmp()
2161 cmn->dtc = devm_kcalloc(cmn->dev, cmn->num_dtcs, sizeof(cmn->dtc[0]), GFP_KERNEL); in arm_cmn_init_dtcs()
2162 if (!cmn->dtc) in arm_cmn_init_dtcs()
2163 return -ENOMEM; in arm_cmn_init_dtcs()
2165 sort(cmn->dns, cmn->num_dns, sizeof(cmn->dns[0]), arm_cmn_node_cmp, NULL); in arm_cmn_init_dtcs()
2167 cmn->xps = arm_cmn_node(cmn, CMN_TYPE_XP); in arm_cmn_init_dtcs()
2169 if (cmn->part == PART_CMN600 && cmn->num_dtcs > 1) { in arm_cmn_init_dtcs()
2172 for (int i = 0; i < cmn->num_dtcs; i++) in arm_cmn_init_dtcs()
2173 arm_cmn_node_to_xp(cmn, dn + i)->dtc = i; in arm_cmn_init_dtcs()
2176 for (dn = cmn->dns; dn->type; dn++) { in arm_cmn_init_dtcs()
2177 if (dn->type == CMN_TYPE_XP) in arm_cmn_init_dtcs()
2181 dn->portid_bits = xp->portid_bits; in arm_cmn_init_dtcs()
2182 dn->deviceid_bits = xp->deviceid_bits; in arm_cmn_init_dtcs()
2183 dn->dtc = xp->dtc; in arm_cmn_init_dtcs()
2184 dn->dtm = xp->dtm; in arm_cmn_init_dtcs()
2185 if (cmn->multi_dtm) in arm_cmn_init_dtcs()
2186 dn->dtm += arm_cmn_nid(dn).port / 2; in arm_cmn_init_dtcs()
2188 if (dn->type == CMN_TYPE_DTC) { in arm_cmn_init_dtcs()
2195 /* To the PMU, RN-Ds don't add anything over RN-Is, so smoosh them together */ in arm_cmn_init_dtcs()
2196 if (dn->type == CMN_TYPE_RND) in arm_cmn_init_dtcs()
2197 dn->type = CMN_TYPE_RNI; in arm_cmn_init_dtcs()
2199 /* We split the RN-I off already, so let the CCLA part match CCLA events */ in arm_cmn_init_dtcs()
2200 if (dn->type == CMN_TYPE_CCLA_RNI) in arm_cmn_init_dtcs()
2201 dn->type = CMN_TYPE_CCLA; in arm_cmn_init_dtcs()
2213 if (cmn->part == PART_CMN650 || cmn->part == PART_CI700) in arm_cmn_dtc_domain()
2222 u64 reg = readq_relaxed(cmn->base + offset + CMN_NODE_INFO); in arm_cmn_init_node_info()
2224 node->type = FIELD_GET(CMN_NI_NODE_TYPE, reg); in arm_cmn_init_node_info()
2225 node->id = FIELD_GET(CMN_NI_NODE_ID, reg); in arm_cmn_init_node_info()
2226 node->logid = FIELD_GET(CMN_NI_LOGICAL_ID, reg); in arm_cmn_init_node_info()
2228 node->pmu_base = cmn->base + offset + arm_cmn_pmu_offset(cmn, node); in arm_cmn_init_node_info()
2230 if (node->type == CMN_TYPE_CFG) in arm_cmn_init_node_info()
2232 else if (node->type == CMN_TYPE_XP) in arm_cmn_init_node_info()
2237 dev_dbg(cmn->dev, "node%*c%#06hx%*ctype:%-#6x id:%-4hd off:%#x\n", in arm_cmn_init_node_info()
2238 (level * 2) + 1, ' ', node->id, 5 - (level * 2), ' ', in arm_cmn_init_node_info()
2239 node->type, node->logid, offset); in arm_cmn_init_node_info()
2268 return -ENODEV; in arm_cmn_discover()
2270 cfg_region = cmn->base + rgn_offset; in arm_cmn_discover()
2275 if (cmn->part && cmn->part != part) in arm_cmn_discover()
2276 dev_warn(cmn->dev, in arm_cmn_discover()
2278 cmn->part, part); in arm_cmn_discover()
2279 cmn->part = part; in arm_cmn_discover()
2281 dev_warn(cmn->dev, "Unknown part number: 0x%x\n", part); in arm_cmn_discover()
2284 cmn->rev = FIELD_GET(CMN_CFGM_PID2_REVISION, reg); in arm_cmn_discover()
2289 * it; however we also have no way to tell from Non-Secure whether any in arm_cmn_discover()
2290 * given port is disabled or not, so the only way to win is not to play... in arm_cmn_discover()
2294 dev_err(cmn->dev, "Device isolation enabled, not continuing due to risk of lockup\n"); in arm_cmn_discover()
2295 return -ENODEV; in arm_cmn_discover()
2297 cmn->multi_dtm = reg & CMN_INFO_MULTIPLE_DTM_EN; in arm_cmn_discover()
2298 cmn->rsp_vc_num = FIELD_GET(CMN_INFO_RSP_VC_NUM, reg); in arm_cmn_discover()
2299 cmn->dat_vc_num = FIELD_GET(CMN_INFO_DAT_VC_NUM, reg); in arm_cmn_discover()
2302 cmn->snp_vc_num = FIELD_GET(CMN_INFO_SNP_VC_NUM, reg); in arm_cmn_discover()
2303 cmn->req_vc_num = FIELD_GET(CMN_INFO_REQ_VC_NUM, reg); in arm_cmn_discover()
2309 cmn->num_xps = child_count; in arm_cmn_discover()
2310 cmn->num_dns = cmn->num_xps; in arm_cmn_discover()
2313 for (i = 0; i < cmn->num_xps; i++) { in arm_cmn_discover()
2317 reg = readq_relaxed(cmn->base + xp_offset[i] + CMN_CHILD_INFO); in arm_cmn_discover()
2318 cmn->num_dns += FIELD_GET(CMN_CI_CHILD_COUNT, reg); in arm_cmn_discover()
2324 * bound, account for double the number of non-XP nodes. in arm_cmn_discover()
2326 dn = devm_kcalloc(cmn->dev, cmn->num_dns * 2 - cmn->num_xps, in arm_cmn_discover()
2329 return -ENOMEM; in arm_cmn_discover()
2332 i = cmn->num_xps; in arm_cmn_discover()
2333 if (cmn->multi_dtm) in arm_cmn_discover()
2334 i += cmn->num_xps + 1; in arm_cmn_discover()
2335 dtm = devm_kcalloc(cmn->dev, i, sizeof(*dtm), GFP_KERNEL); in arm_cmn_discover()
2337 return -ENOMEM; in arm_cmn_discover()
2340 cmn->dns = dn; in arm_cmn_discover()
2341 cmn->dtms = dtm; in arm_cmn_discover()
2342 for (i = 0; i < cmn->num_xps; i++) { in arm_cmn_discover()
2343 void __iomem *xp_region = cmn->base + xp_offset[i]; in arm_cmn_discover()
2354 if (xp->id == (1 << 3)) in arm_cmn_discover()
2355 cmn->mesh_x = xp->logid; in arm_cmn_discover()
2357 if (cmn->part == PART_CMN600) in arm_cmn_discover()
2358 xp->dtc = -1; in arm_cmn_discover()
2360 xp->dtc = arm_cmn_dtc_domain(cmn, xp_region); in arm_cmn_discover()
2362 xp->dtm = dtm - cmn->dtms; in arm_cmn_discover()
2366 * unnecessary XP events easily, and also infer the per-XP in arm_cmn_discover()
2373 if (cmn->num_xps == 1) { in arm_cmn_discover()
2374 xp->portid_bits = 3; in arm_cmn_discover()
2375 xp->deviceid_bits = 2; in arm_cmn_discover()
2377 xp->portid_bits = 2; in arm_cmn_discover()
2378 xp->deviceid_bits = 1; in arm_cmn_discover()
2380 xp->portid_bits = 1; in arm_cmn_discover()
2381 xp->deviceid_bits = 2; in arm_cmn_discover()
2384 if (cmn->multi_dtm && (xp_ports > 0x3)) in arm_cmn_discover()
2386 if (cmn->multi_dtm && (xp_ports > 0xf)) in arm_cmn_discover()
2389 cmn->ports_used |= xp_ports; in arm_cmn_discover()
2400 * As of CMN-600r1 these could only be RN-SAMs or CXLAs, in arm_cmn_discover()
2407 dev_dbg(cmn->dev, "ignoring external node %llx\n", reg); in arm_cmn_discover()
2413 * A child offset of 0 can only occur on CMN-600; otherwise it in arm_cmn_discover()
2417 if (reg == 0 && cmn->part != PART_CMN600) { in arm_cmn_discover()
2418 dev_dbg(cmn->dev, "bogus child pointer?\n"); in arm_cmn_discover()
2424 switch (dn->type) { in arm_cmn_discover()
2426 cmn->num_dtcs++; in arm_cmn_discover()
2445 dn->pmu_base += CMN_CCLA_PMU_EVENT_SEL; in arm_cmn_discover()
2467 dn[1].type = arm_cmn_subtype(dn->type); in arm_cmn_discover()
2472 dev_err(cmn->dev, "invalid device node type: 0x%x\n", dn->type); in arm_cmn_discover()
2473 return -ENODEV; in arm_cmn_discover()
2479 cmn->num_dns = dn - cmn->dns; in arm_cmn_discover()
2481 /* Cheeky +1 to help terminate pointer-based iteration later */ in arm_cmn_discover()
2482 sz = (void *)(dn + 1) - (void *)cmn->dns; in arm_cmn_discover()
2483 dn = devm_krealloc(cmn->dev, cmn->dns, sz, GFP_KERNEL); in arm_cmn_discover()
2485 cmn->dns = dn; in arm_cmn_discover()
2487 sz = (void *)dtm - (void *)cmn->dtms; in arm_cmn_discover()
2488 dtm = devm_krealloc(cmn->dev, cmn->dtms, sz, GFP_KERNEL); in arm_cmn_discover()
2490 cmn->dtms = dtm; in arm_cmn_discover()
2496 if (!cmn->mesh_x) in arm_cmn_discover()
2497 cmn->mesh_x = cmn->num_xps; in arm_cmn_discover()
2498 cmn->mesh_y = cmn->num_xps / cmn->mesh_x; in arm_cmn_discover()
2501 if (cmn->num_xps == 1) in arm_cmn_discover()
2502 dev_warn(cmn->dev, "1x1 config not fully supported, translate XP events manually\n"); in arm_cmn_discover()
2504 dev_dbg(cmn->dev, "periph_id part 0x%03x revision %d\n", cmn->part, cmn->rev); in arm_cmn_discover()
2505 reg = cmn->ports_used; in arm_cmn_discover()
2506 dev_dbg(cmn->dev, "mesh %dx%d, ID width %d, ports %6pbl%s\n", in arm_cmn_discover()
2507 cmn->mesh_x, cmn->mesh_y, arm_cmn_xyidbits(cmn), ®, in arm_cmn_discover()
2508 cmn->multi_dtm ? ", multi-DTM" : ""); in arm_cmn_discover()
2519 return -EINVAL; in arm_cmn600_acpi_probe()
2523 return -EINVAL; in arm_cmn600_acpi_probe()
2533 cmn->base = devm_ioremap(cmn->dev, cfg->start, resource_size(cfg)); in arm_cmn600_acpi_probe()
2534 if (!cmn->base) in arm_cmn600_acpi_probe()
2535 return -ENOMEM; in arm_cmn600_acpi_probe()
2537 return root->start - cfg->start; in arm_cmn600_acpi_probe()
2544 return of_property_read_u32(np, "arm,root-node", &rootnode) ?: rootnode; in arm_cmn600_of_probe()
2554 cmn = devm_kzalloc(&pdev->dev, sizeof(*cmn), GFP_KERNEL); in arm_cmn_probe()
2556 return -ENOMEM; in arm_cmn_probe()
2558 cmn->dev = &pdev->dev; in arm_cmn_probe()
2559 cmn->part = (unsigned long)device_get_match_data(cmn->dev); in arm_cmn_probe()
2562 if (cmn->part == PART_CMN600 && has_acpi_companion(cmn->dev)) { in arm_cmn_probe()
2566 cmn->base = devm_platform_ioremap_resource(pdev, 0); in arm_cmn_probe()
2567 if (IS_ERR(cmn->base)) in arm_cmn_probe()
2568 return PTR_ERR(cmn->base); in arm_cmn_probe()
2569 if (cmn->part == PART_CMN600) in arm_cmn_probe()
2570 rootnode = arm_cmn600_of_probe(pdev->dev.of_node); in arm_cmn_probe()
2587 cmn->cpu = cpumask_local_spread(0, dev_to_node(cmn->dev)); in arm_cmn_probe()
2588 cmn->pmu = (struct pmu) { in arm_cmn_probe()
2590 .parent = cmn->dev, in arm_cmn_probe()
2608 name = devm_kasprintf(cmn->dev, GFP_KERNEL, "arm_cmn_%d", this_id); in arm_cmn_probe()
2610 return -ENOMEM; in arm_cmn_probe()
2612 err = cpuhp_state_add_instance(arm_cmn_hp_state, &cmn->cpuhp_node); in arm_cmn_probe()
2616 err = perf_pmu_register(&cmn->pmu, name, -1); in arm_cmn_probe()
2618 cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node); in arm_cmn_probe()
2629 writel_relaxed(0, cmn->dtc[0].base + CMN_DT_DTC_CTL); in arm_cmn_remove()
2631 perf_pmu_unregister(&cmn->pmu); in arm_cmn_remove()
2632 cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node); in arm_cmn_remove()
2633 debugfs_remove(cmn->debug); in arm_cmn_remove()
2638 { .compatible = "arm,cmn-600", .data = (void *)PART_CMN600 },
2639 { .compatible = "arm,cmn-650" },
2640 { .compatible = "arm,cmn-700" },
2641 { .compatible = "arm,cmn-s3" },
2642 { .compatible = "arm,ci-700" },
2660 .name = "arm-cmn",
2680 arm_cmn_debugfs = debugfs_create_dir("arm-cmn", NULL); in arm_cmn_init()
2701 MODULE_DESCRIPTION("Arm CMN-600 PMU driver");