Lines Matching +full:armv8 +full:- +full:based
1 // SPDX-License-Identifier: GPL-2.0-only
3 * ARMv8 PMUv3 Performance Events handling code.
8 * This code is based heavily on the ARMv7 perf event code.
28 /* ARMv8 Cortex-A53 specific event types. */
31 /* ARMv8 Cavium ThunderX specific event types. */
39 * ARMv8 Architectural defined events, not all of these may
41 * be disabled at run-time based on the PMCEID registers.
164 return sprintf(page, "event=0x%04llx\n", pmu_attr->id); in armv8pmu_events_sysfs_show()
174 * means we don't have a fixed event<->counter relationship regardless.
279 if (pmu_attr->id < ARMV8_PMUV3_MAX_COMMON_EVENTS && in armv8pmu_event_attr_is_visible()
280 test_bit(pmu_attr->id, cpu_pmu->pmceid_bitmap)) in armv8pmu_event_attr_is_visible()
281 return attr->mode; in armv8pmu_event_attr_is_visible()
283 if (pmu_attr->id >= ARMV8_PMUV3_EXT_COMMON_EVENT_BASE) { in armv8pmu_event_attr_is_visible()
284 u64 id = pmu_attr->id - ARMV8_PMUV3_EXT_COMMON_EVENT_BASE; in armv8pmu_event_attr_is_visible()
287 test_bit(id, cpu_pmu->pmceid_ext_bitmap)) in armv8pmu_event_attr_is_visible()
288 return attr->mode; in armv8pmu_event_attr_is_visible()
331 return ATTR_CFG_GET_FLD(&event->attr, long); in armv8pmu_event_is_64bit()
336 return ATTR_CFG_GET_FLD(&event->attr, rdpmc); in armv8pmu_event_want_user_access()
378 u32 slots = FIELD_GET(ARMV8_PMU_SLOTS, cpu_pmu->reg_pmmir); in slots_show()
390 u32 bus_slots = FIELD_GET(ARMV8_PMU_BUS_SLOTS, cpu_pmu->reg_pmmir); in bus_slots_show()
402 u32 bus_width = FIELD_GET(ARMV8_PMU_BUS_WIDTH, cpu_pmu->reg_pmmir); in bus_width_show()
407 val = 1 << (bus_width - 1); in bus_width_show()
417 * PMMIR.THWIDTH is readable and non-zero on aarch32, but it would be in threshold_max()
425 * (2 ^ PMMIR.THWIDTH) - 1. in threshold_max()
427 return (1 << FIELD_GET(ARMV8_PMU_THWIDTH, cpu_pmu->reg_pmmir)) - 1; in threshold_max()
455 * We unconditionally enable ARMv8.5-PMU long event counter support
456 * (64-bit events) where supported. Indicate if this arm_pmu has long
464 return (IS_ENABLED(CONFIG_ARM64) && is_pmuv3p5(cpu_pmu->pmuver)); in armv8pmu_has_long_event()
469 return event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT; in armv8pmu_event_has_user_read()
479 int idx = event->hw.idx; in armv8pmu_event_is_chained()
480 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); in armv8pmu_event_is_chained()
489 * ARMv8 low level PMU access
520 int idx = event->hw.idx; in armv8pmu_read_hw_counter()
524 val = (val << 32) | armv8pmu_read_evcntr(idx - 1); in armv8pmu_read_hw_counter()
529 * The cycle counter is always a 64-bit counter. When ARMV8_PMU_PMCR_LP
530 * is set the event counters also become 64-bit counters. Unless the
532 * interrupt upon 32-bit overflow - we achieve this by applying a bias.
536 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); in armv8pmu_event_needs_bias()
537 struct hw_perf_event *hwc = &event->hw; in armv8pmu_event_needs_bias()
538 int idx = hwc->idx; in armv8pmu_event_needs_bias()
568 struct hw_perf_event *hwc = &event->hw; in armv8pmu_read_counter()
569 int idx = hwc->idx; in armv8pmu_read_counter()
590 int idx = event->hw.idx; in armv8pmu_write_hw_counter()
594 armv8pmu_write_evcntr(idx - 1, lower_32_bits(value)); in armv8pmu_write_hw_counter()
602 struct hw_perf_event *hwc = &event->hw; in armv8pmu_write_counter()
603 int idx = hwc->idx; in armv8pmu_write_counter()
631 struct hw_perf_event *hwc = &event->hw; in armv8pmu_write_event_type()
632 int idx = hwc->idx; in armv8pmu_write_event_type()
643 armv8pmu_write_evtype(idx - 1, hwc->config_base); in armv8pmu_write_event_type()
647 write_pmccfiltr(hwc->config_base); in armv8pmu_write_event_type()
649 write_pmicfiltr(hwc->config_base); in armv8pmu_write_event_type()
651 armv8pmu_write_evtype(idx, hwc->config_base); in armv8pmu_write_event_type()
657 int counter = event->hw.idx; in armv8pmu_event_cnten_mask()
661 mask |= BIT(counter - 1); in armv8pmu_event_cnten_mask()
677 struct perf_event_attr *attr = &event->attr; in armv8pmu_enable_event_counter()
699 struct perf_event_attr *attr = &event->attr; in armv8pmu_disable_event_counter()
716 armv8pmu_enable_intens(BIT(event->hw.idx)); in armv8pmu_enable_event_irq()
730 armv8pmu_disable_intens(BIT(event->hw.idx)); in armv8pmu_disable_event_irq()
771 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); in armv8pmu_enable_user_access()
774 for_each_andnot_bit(i, cpu_pmu->cntr_mask, cpuc->used_mask, in armv8pmu_enable_user_access()
812 nr_user = ctx->nr_user; in armv8pmu_start()
835 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); in armv8pmu_handle_irq()
860 for_each_set_bit(idx, cpu_pmu->cntr_mask, ARMPMU_MAX_HWEVENTS) { in armv8pmu_handle_irq()
861 struct perf_event *event = cpuc->events[idx]; in armv8pmu_handle_irq()
875 hwc = &event->hw; in armv8pmu_handle_irq()
877 perf_sample_data_init(&data, 0, hwc->last_period); in armv8pmu_handle_irq()
887 cpu_pmu->disable(event); in armv8pmu_handle_irq()
899 for_each_set_bit(idx, cpu_pmu->cntr_mask, ARMV8_PMU_MAX_GENERAL_COUNTERS) { in armv8pmu_get_single_idx()
900 if (!test_and_set_bit(idx, cpuc->used_mask)) in armv8pmu_get_single_idx()
903 return -EAGAIN; in armv8pmu_get_single_idx()
915 for_each_set_bit(idx, cpu_pmu->cntr_mask, ARMV8_PMU_MAX_GENERAL_COUNTERS) { in armv8pmu_get_chain_idx()
918 if (!test_and_set_bit(idx, cpuc->used_mask)) { in armv8pmu_get_chain_idx()
920 if (!test_and_set_bit(idx - 1, cpuc->used_mask)) in armv8pmu_get_chain_idx()
923 clear_bit(idx, cpuc->used_mask); in armv8pmu_get_chain_idx()
926 return -EAGAIN; in armv8pmu_get_chain_idx()
932 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); in armv8pmu_get_event_idx()
933 struct hw_perf_event *hwc = &event->hw; in armv8pmu_get_event_idx()
934 unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT; in armv8pmu_get_event_idx()
938 !armv8pmu_event_get_threshold(&event->attr)) { in armv8pmu_get_event_idx()
939 if (!test_and_set_bit(ARMV8_PMU_CYCLE_IDX, cpuc->used_mask)) in armv8pmu_get_event_idx()
944 return -EAGAIN; in armv8pmu_get_event_idx()
953 !armv8pmu_event_get_threshold(&event->attr) && in armv8pmu_get_event_idx()
954 test_bit(ARMV8_PMU_INSTR_IDX, cpu_pmu->cntr_mask) && in armv8pmu_get_event_idx()
956 if (!test_and_set_bit(ARMV8_PMU_INSTR_IDX, cpuc->used_mask)) in armv8pmu_get_event_idx()
972 int idx = event->hw.idx; in armv8pmu_clear_event_idx()
974 clear_bit(idx, cpuc->used_mask); in armv8pmu_clear_event_idx()
976 clear_bit(idx - 1, cpuc->used_mask); in armv8pmu_clear_event_idx()
984 return event->hw.idx + 1; in armv8pmu_user_event_idx()
996 struct arm_pmu *cpu_pmu = to_arm_pmu(perf_event->pmu); in armv8pmu_set_event_filter()
999 if (attr->exclude_idle) { in armv8pmu_set_event_filter()
1001 return -EOPNOTSUPP; in armv8pmu_set_event_filter()
1011 if (!attr->exclude_kernel && !attr->exclude_host) in armv8pmu_set_event_filter()
1013 if (attr->exclude_guest) in armv8pmu_set_event_filter()
1015 if (attr->exclude_host) in armv8pmu_set_event_filter()
1018 if (!attr->exclude_hv && !attr->exclude_host) in armv8pmu_set_event_filter()
1025 if (attr->exclude_kernel) in armv8pmu_set_event_filter()
1028 if (attr->exclude_user) in armv8pmu_set_event_filter()
1038 return -EINVAL; in armv8pmu_set_event_filter()
1051 event->config_base = config_base; in armv8pmu_set_event_filter()
1061 bitmap_to_arr64(&mask, cpu_pmu->cntr_mask, ARMPMU_MAX_HWEVENTS); in armv8pmu_reset()
1086 if (event->attr.type == PERF_TYPE_HARDWARE && in __armv8_pmuv3_map_event_id()
1087 event->attr.config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) { in __armv8_pmuv3_map_event_id()
1090 armpmu->pmceid_bitmap)) in __armv8_pmuv3_map_event_id()
1094 armpmu->pmceid_bitmap)) in __armv8_pmuv3_map_event_id()
1114 struct arm_pmu *armpmu = to_arm_pmu(event->pmu); in __armv8_pmuv3_map_event()
1124 return -EINVAL; in __armv8_pmuv3_map_event()
1127 event->hw.flags |= ARMPMU_EVT_64BIT; in __armv8_pmuv3_map_event()
1133 * Most 64-bit events require long counter support, but 64-bit in __armv8_pmuv3_map_event()
1138 if (!(event->attach_state & PERF_ATTACH_TASK)) in __armv8_pmuv3_map_event()
1139 return -EINVAL; in __armv8_pmuv3_map_event()
1143 return -EOPNOTSUPP; in __armv8_pmuv3_map_event()
1145 event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT; in __armv8_pmuv3_map_event()
1150 && test_bit(hw_event_id, armpmu->pmceid_bitmap)) { in __armv8_pmuv3_map_event()
1198 struct arm_pmu *cpu_pmu = probe->pmu; in __armv8pmu_probe_pmu()
1207 cpu_pmu->pmuver = pmuver; in __armv8pmu_probe_pmu()
1208 probe->present = true; in __armv8pmu_probe_pmu()
1211 bitmap_set(cpu_pmu->cntr_mask, in __armv8pmu_probe_pmu()
1215 set_bit(ARMV8_PMU_CYCLE_IDX, cpu_pmu->cntr_mask); in __armv8pmu_probe_pmu()
1219 set_bit(ARMV8_PMU_INSTR_IDX, cpu_pmu->cntr_mask); in __armv8pmu_probe_pmu()
1224 bitmap_from_arr32(cpu_pmu->pmceid_bitmap, in __armv8pmu_probe_pmu()
1230 bitmap_from_arr32(cpu_pmu->pmceid_ext_bitmap, in __armv8pmu_probe_pmu()
1235 cpu_pmu->reg_pmmir = read_pmmir(); in __armv8pmu_probe_pmu()
1237 cpu_pmu->reg_pmmir = 0; in __armv8pmu_probe_pmu()
1248 ret = smp_call_function_any(&cpu_pmu->supported_cpus, in armv8pmu_probe_pmu()
1254 return probe.present ? 0 : -ENODEV; in armv8pmu_probe_pmu()
1300 cpu_pmu->handle_irq = armv8pmu_handle_irq; in armv8_pmu_init()
1301 cpu_pmu->enable = armv8pmu_enable_event; in armv8_pmu_init()
1302 cpu_pmu->disable = armv8pmu_disable_event; in armv8_pmu_init()
1303 cpu_pmu->read_counter = armv8pmu_read_counter; in armv8_pmu_init()
1304 cpu_pmu->write_counter = armv8pmu_write_counter; in armv8_pmu_init()
1305 cpu_pmu->get_event_idx = armv8pmu_get_event_idx; in armv8_pmu_init()
1306 cpu_pmu->clear_event_idx = armv8pmu_clear_event_idx; in armv8_pmu_init()
1307 cpu_pmu->start = armv8pmu_start; in armv8_pmu_init()
1308 cpu_pmu->stop = armv8pmu_stop; in armv8_pmu_init()
1309 cpu_pmu->reset = armv8pmu_reset; in armv8_pmu_init()
1310 cpu_pmu->set_event_filter = armv8pmu_set_event_filter; in armv8_pmu_init()
1312 cpu_pmu->pmu.event_idx = armv8pmu_user_event_idx; in armv8_pmu_init()
1314 cpu_pmu->name = name; in armv8_pmu_init()
1315 cpu_pmu->map_event = map_event; in armv8_pmu_init()
1316 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = &armv8_pmuv3_events_attr_group; in armv8_pmu_init()
1317 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = &armv8_pmuv3_format_attr_group; in armv8_pmu_init()
1318 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_CAPS] = &armv8_pmuv3_caps_attr_group; in armv8_pmu_init()
1376 {.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_pmu_init},
1377 {.compatible = "arm,cortex-a34-pmu", .data = armv8_cortex_a34_pmu_init},
1378 {.compatible = "arm,cortex-a35-pmu", .data = armv8_cortex_a35_pmu_init},
1379 {.compatible = "arm,cortex-a53-pmu", .data = armv8_cortex_a53_pmu_init},
1380 {.compatible = "arm,cortex-a55-pmu", .data = armv8_cortex_a55_pmu_init},
1381 {.compatible = "arm,cortex-a57-pmu", .data = armv8_cortex_a57_pmu_init},
1382 {.compatible = "arm,cortex-a65-pmu", .data = armv8_cortex_a65_pmu_init},
1383 {.compatible = "arm,cortex-a72-pmu", .data = armv8_cortex_a72_pmu_init},
1384 {.compatible = "arm,cortex-a73-pmu", .data = armv8_cortex_a73_pmu_init},
1385 {.compatible = "arm,cortex-a75-pmu", .data = armv8_cortex_a75_pmu_init},
1386 {.compatible = "arm,cortex-a76-pmu", .data = armv8_cortex_a76_pmu_init},
1387 {.compatible = "arm,cortex-a77-pmu", .data = armv8_cortex_a77_pmu_init},
1388 {.compatible = "arm,cortex-a78-pmu", .data = armv8_cortex_a78_pmu_init},
1389 {.compatible = "arm,cortex-a510-pmu", .data = armv9_cortex_a510_pmu_init},
1390 {.compatible = "arm,cortex-a520-pmu", .data = armv9_cortex_a520_pmu_init},
1391 {.compatible = "arm,cortex-a710-pmu", .data = armv9_cortex_a710_pmu_init},
1392 {.compatible = "arm,cortex-a715-pmu", .data = armv9_cortex_a715_pmu_init},
1393 {.compatible = "arm,cortex-a720-pmu", .data = armv9_cortex_a720_pmu_init},
1394 {.compatible = "arm,cortex-a725-pmu", .data = armv9_cortex_a725_pmu_init},
1395 {.compatible = "arm,cortex-x1-pmu", .data = armv8_cortex_x1_pmu_init},
1396 {.compatible = "arm,cortex-x2-pmu", .data = armv9_cortex_x2_pmu_init},
1397 {.compatible = "arm,cortex-x3-pmu", .data = armv9_cortex_x3_pmu_init},
1398 {.compatible = "arm,cortex-x4-pmu", .data = armv9_cortex_x4_pmu_init},
1399 {.compatible = "arm,cortex-x925-pmu", .data = armv9_cortex_x925_pmu_init},
1400 {.compatible = "arm,neoverse-e1-pmu", .data = armv8_neoverse_e1_pmu_init},
1401 {.compatible = "arm,neoverse-n1-pmu", .data = armv8_neoverse_n1_pmu_init},
1402 {.compatible = "arm,neoverse-n2-pmu", .data = armv9_neoverse_n2_pmu_init},
1403 {.compatible = "arm,neoverse-n3-pmu", .data = armv9_neoverse_n3_pmu_init},
1404 {.compatible = "arm,neoverse-v1-pmu", .data = armv8_neoverse_v1_pmu_init},
1405 {.compatible = "arm,neoverse-v2-pmu", .data = armv8_neoverse_v2_pmu_init},
1406 {.compatible = "arm,neoverse-v3-pmu", .data = armv8_neoverse_v3_pmu_init},
1407 {.compatible = "arm,neoverse-v3ae-pmu", .data = armv8_neoverse_v3ae_pmu_init},
1408 {.compatible = "cavium,thunder-pmu", .data = armv8_cavium_thunder_pmu_init},
1409 {.compatible = "brcm,vulcan-pmu", .data = armv8_brcm_vulcan_pmu_init},
1410 {.compatible = "nvidia,carmel-pmu", .data = armv8_nvidia_carmel_pmu_init},
1411 {.compatible = "nvidia,denver-pmu", .data = armv8_nvidia_denver_pmu_init},
1452 userpg->cap_user_time = 0; in device_initcall()
1453 userpg->cap_user_time_zero = 0; in device_initcall()
1454 userpg->cap_user_time_short = 0; in device_initcall()
1455 userpg->cap_user_rdpmc = armv8pmu_event_has_user_read(event); in device_initcall()
1457 if (userpg->cap_user_rdpmc) { in device_initcall()
1458 if (event->hw.flags & ARMPMU_EVT_64BIT) in device_initcall()
1459 userpg->pmc_width = 64; in device_initcall()
1461 userpg->pmc_width = 32; in device_initcall()
1467 if (rd->read_sched_clock != arch_timer_read_counter) in device_initcall()
1470 userpg->time_mult = rd->mult; in device_initcall()
1471 userpg->time_shift = rd->shift; in device_initcall()
1472 userpg->time_zero = rd->epoch_ns; in device_initcall()
1473 userpg->time_cycles = rd->epoch_cyc; in device_initcall()
1474 userpg->time_mask = rd->sched_clock_mask; in device_initcall()
1481 ns = mul_u64_u32_shr(rd->epoch_cyc, rd->mult, rd->shift); in device_initcall()
1482 userpg->time_zero -= ns; in device_initcall()
1486 userpg->time_offset = userpg->time_zero - now; in device_initcall()
1491 * 32-bit value (now specifies a 64-bit value) - refer in device_initcall()
1494 if (userpg->time_shift == 32) { in device_initcall()
1495 userpg->time_shift = 31; in device_initcall()
1496 userpg->time_mult >>= 1; in device_initcall()
1503 userpg->cap_user_time = 1; in device_initcall()
1504 userpg->cap_user_time_zero = 1; in device_initcall()
1505 userpg->cap_user_time_short = 1; in device_initcall()