Lines Matching +full:cpu +full:- +full:cfg
1 // SPDX-License-Identifier: GPL-2.0-only
6 * Intel RAPL interface is specified in the IA-32 Manual Vol3b
34 * gpu counter: consumption of the builtin-gpu domain (client only)
38 * psys counter: consumption of the builtin-psys domain (client only)
42 * We manage those counters as free running (read-only). They may be
45 * The events only support system-wide mode counting. There is no
49 * Because we want to avoid floating-point operations in the kernel,
53 * ldexp(raw_count, -32);
63 #include <asm/intel-family.h>
85 "pp0-core",
88 "pp1-gpu",
93 * event code: LSB 8 bits, passed in attr->config
108 * 1. AMD/HYGON platforms have a per-PKG package energy counter
110 * 2.1. CLX-AP is multi-die and its RAPL MSRs are die-scope
112 * considered as either pkg-scope or die-scope, and we are considering
113 * them as die-scope.
122 int cpu; member
160 static inline unsigned int get_rapl_pmu_idx(int cpu) in get_rapl_pmu_idx() argument
162 return rapl_pmu_is_pkg_scope() ? topology_logical_package_id(cpu) : in get_rapl_pmu_idx()
163 topology_logical_die_id(cpu); in get_rapl_pmu_idx()
166 static inline const struct cpumask *get_rapl_pmu_cpumask(int cpu) in get_rapl_pmu_cpumask() argument
168 return rapl_pmu_is_pkg_scope() ? topology_core_cpumask(cpu) : in get_rapl_pmu_cpumask()
169 topology_die_cpumask(cpu); in get_rapl_pmu_cpumask()
172 static inline struct rapl_pmu *cpu_to_rapl_pmu(unsigned int cpu) in cpu_to_rapl_pmu() argument
174 unsigned int rapl_pmu_idx = get_rapl_pmu_idx(cpu); in cpu_to_rapl_pmu()
177 * The unsigned check also catches the '-1' return value for non in cpu_to_rapl_pmu()
180 return rapl_pmu_idx < rapl_pmus->nr_rapl_pmu ? rapl_pmus->pmus[rapl_pmu_idx] : NULL; in cpu_to_rapl_pmu()
186 rdmsrl(event->hw.event_base, raw); in rapl_read_counter()
190 static inline u64 rapl_scale(u64 v, int cfg) in rapl_scale() argument
192 if (cfg > NR_RAPL_DOMAINS) { in rapl_scale()
193 pr_warn("Invalid domain %d, failed to scale data\n", cfg); in rapl_scale()
199 * or use ldexp(count, -32). in rapl_scale()
202 return v << (32 - rapl_hw_unit[cfg - 1]); in rapl_scale()
207 struct hw_perf_event *hwc = &event->hw; in rapl_event_update()
212 prev_raw_count = local64_read(&hwc->prev_count); in rapl_event_update()
214 rdmsrl(event->hw.event_base, new_raw_count); in rapl_event_update()
215 } while (!local64_try_cmpxchg(&hwc->prev_count, in rapl_event_update()
221 * (event-)time and add that to the generic event. in rapl_event_update()
223 * Careful, not all hw sign-extends above the physical width in rapl_event_update()
226 delta = (new_raw_count << shift) - (prev_raw_count << shift); in rapl_event_update()
229 sdelta = rapl_scale(delta, event->hw.config); in rapl_event_update()
231 local64_add(sdelta, &event->count); in rapl_event_update()
238 hrtimer_start(&pmu->hrtimer, pmu->timer_interval, in rapl_start_hrtimer()
248 if (!pmu->n_active) in rapl_hrtimer_handle()
251 raw_spin_lock_irqsave(&pmu->lock, flags); in rapl_hrtimer_handle()
253 list_for_each_entry(event, &pmu->active_list, active_entry) in rapl_hrtimer_handle()
256 raw_spin_unlock_irqrestore(&pmu->lock, flags); in rapl_hrtimer_handle()
258 hrtimer_forward_now(hrtimer, pmu->timer_interval); in rapl_hrtimer_handle()
265 struct hrtimer *hr = &pmu->hrtimer; in rapl_hrtimer_init()
268 hr->function = rapl_hrtimer_handle; in rapl_hrtimer_init()
274 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) in __rapl_pmu_event_start()
277 event->hw.state = 0; in __rapl_pmu_event_start()
279 list_add_tail(&event->active_entry, &pmu->active_list); in __rapl_pmu_event_start()
281 local64_set(&event->hw.prev_count, rapl_read_counter(event)); in __rapl_pmu_event_start()
283 pmu->n_active++; in __rapl_pmu_event_start()
284 if (pmu->n_active == 1) in __rapl_pmu_event_start()
290 struct rapl_pmu *pmu = event->pmu_private; in rapl_pmu_event_start()
293 raw_spin_lock_irqsave(&pmu->lock, flags); in rapl_pmu_event_start()
295 raw_spin_unlock_irqrestore(&pmu->lock, flags); in rapl_pmu_event_start()
300 struct rapl_pmu *pmu = event->pmu_private; in rapl_pmu_event_stop()
301 struct hw_perf_event *hwc = &event->hw; in rapl_pmu_event_stop()
304 raw_spin_lock_irqsave(&pmu->lock, flags); in rapl_pmu_event_stop()
307 if (!(hwc->state & PERF_HES_STOPPED)) { in rapl_pmu_event_stop()
308 WARN_ON_ONCE(pmu->n_active <= 0); in rapl_pmu_event_stop()
309 pmu->n_active--; in rapl_pmu_event_stop()
310 if (pmu->n_active == 0) in rapl_pmu_event_stop()
311 hrtimer_cancel(&pmu->hrtimer); in rapl_pmu_event_stop()
313 list_del(&event->active_entry); in rapl_pmu_event_stop()
315 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); in rapl_pmu_event_stop()
316 hwc->state |= PERF_HES_STOPPED; in rapl_pmu_event_stop()
320 if ((mode & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { in rapl_pmu_event_stop()
326 hwc->state |= PERF_HES_UPTODATE; in rapl_pmu_event_stop()
329 raw_spin_unlock_irqrestore(&pmu->lock, flags); in rapl_pmu_event_stop()
334 struct rapl_pmu *pmu = event->pmu_private; in rapl_pmu_event_add()
335 struct hw_perf_event *hwc = &event->hw; in rapl_pmu_event_add()
338 raw_spin_lock_irqsave(&pmu->lock, flags); in rapl_pmu_event_add()
340 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; in rapl_pmu_event_add()
345 raw_spin_unlock_irqrestore(&pmu->lock, flags); in rapl_pmu_event_add()
357 u64 cfg = event->attr.config & RAPL_EVENT_MASK; in rapl_pmu_event_init() local
362 if (event->attr.type != rapl_pmus->pmu.type) in rapl_pmu_event_init()
363 return -ENOENT; in rapl_pmu_event_init()
366 if (event->attr.config & ~RAPL_EVENT_MASK) in rapl_pmu_event_init()
367 return -EINVAL; in rapl_pmu_event_init()
369 if (event->cpu < 0) in rapl_pmu_event_init()
370 return -EINVAL; in rapl_pmu_event_init()
372 event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG; in rapl_pmu_event_init()
374 if (!cfg || cfg >= NR_RAPL_DOMAINS + 1) in rapl_pmu_event_init()
375 return -EINVAL; in rapl_pmu_event_init()
377 cfg = array_index_nospec((long)cfg, NR_RAPL_DOMAINS + 1); in rapl_pmu_event_init()
378 bit = cfg - 1; in rapl_pmu_event_init()
382 return -EINVAL; in rapl_pmu_event_init()
385 if (event->attr.sample_period) /* no sampling */ in rapl_pmu_event_init()
386 return -EINVAL; in rapl_pmu_event_init()
389 pmu = cpu_to_rapl_pmu(event->cpu); in rapl_pmu_event_init()
391 return -EINVAL; in rapl_pmu_event_init()
392 event->cpu = pmu->cpu; in rapl_pmu_event_init()
393 event->pmu_private = pmu; in rapl_pmu_event_init()
394 event->hw.event_base = rapl_msrs[bit].msr; in rapl_pmu_event_init()
395 event->hw.config = cfg; in rapl_pmu_event_init()
396 event->hw.idx = bit; in rapl_pmu_event_init()
423 RAPL_EVENT_ATTR_STR(energy-cores, rapl_cores, "event=0x01");
424 RAPL_EVENT_ATTR_STR(energy-pkg , rapl_pkg, "event=0x02");
425 RAPL_EVENT_ATTR_STR(energy-ram , rapl_ram, "event=0x03");
426 RAPL_EVENT_ATTR_STR(energy-gpu , rapl_gpu, "event=0x04");
427 RAPL_EVENT_ATTR_STR(energy-psys, rapl_psys, "event=0x05");
429 RAPL_EVENT_ATTR_STR(energy-cores.unit, rapl_cores_unit, "Joules");
430 RAPL_EVENT_ATTR_STR(energy-pkg.unit , rapl_pkg_unit, "Joules");
431 RAPL_EVENT_ATTR_STR(energy-ram.unit , rapl_ram_unit, "Joules");
432 RAPL_EVENT_ATTR_STR(energy-gpu.unit , rapl_gpu_unit, "Joules");
433 RAPL_EVENT_ATTR_STR(energy-psys.unit, rapl_psys_unit, "Joules");
438 RAPL_EVENT_ATTR_STR(energy-cores.scale, rapl_cores_scale, "2.3283064365386962890625e-10");
439 RAPL_EVENT_ATTR_STR(energy-pkg.scale, rapl_pkg_scale, "2.3283064365386962890625e-10");
440 RAPL_EVENT_ATTR_STR(energy-ram.scale, rapl_ram_scale, "2.3283064365386962890625e-10");
441 RAPL_EVENT_ATTR_STR(energy-gpu.scale, rapl_gpu_scale, "2.3283064365386962890625e-10");
442 RAPL_EVENT_ATTR_STR(energy-psys.scale, rapl_psys_scale, "2.3283064365386962890625e-10");
458 PMU_FORMAT_ATTR(event, "config:0-7");
562 * - perf_msr_probe(PERF_RAPL_MAX)
563 * - want to use same event codes across both architectures
573 static int rapl_cpu_offline(unsigned int cpu) in rapl_cpu_offline() argument
575 struct rapl_pmu *pmu = cpu_to_rapl_pmu(cpu); in rapl_cpu_offline()
578 /* Check if exiting cpu is used for collecting rapl events */ in rapl_cpu_offline()
579 if (!cpumask_test_and_clear_cpu(cpu, &rapl_cpu_mask)) in rapl_cpu_offline()
582 pmu->cpu = -1; in rapl_cpu_offline()
583 /* Find a new cpu to collect rapl events */ in rapl_cpu_offline()
584 target = cpumask_any_but(get_rapl_pmu_cpumask(cpu), cpu); in rapl_cpu_offline()
589 pmu->cpu = target; in rapl_cpu_offline()
590 perf_pmu_migrate_context(pmu->pmu, cpu, target); in rapl_cpu_offline()
595 static int rapl_cpu_online(unsigned int cpu) in rapl_cpu_online() argument
597 s32 rapl_pmu_idx = get_rapl_pmu_idx(cpu); in rapl_cpu_online()
600 return -EINVAL; in rapl_cpu_online()
602 struct rapl_pmu *pmu = cpu_to_rapl_pmu(cpu); in rapl_cpu_online()
606 pmu = kzalloc_node(sizeof(*pmu), GFP_KERNEL, cpu_to_node(cpu)); in rapl_cpu_online()
608 return -ENOMEM; in rapl_cpu_online()
610 raw_spin_lock_init(&pmu->lock); in rapl_cpu_online()
611 INIT_LIST_HEAD(&pmu->active_list); in rapl_cpu_online()
612 pmu->pmu = &rapl_pmus->pmu; in rapl_cpu_online()
613 pmu->timer_interval = ms_to_ktime(rapl_timer_ms); in rapl_cpu_online()
616 rapl_pmus->pmus[rapl_pmu_idx] = pmu; in rapl_cpu_online()
620 * Check if there is an online cpu in the package which collects rapl in rapl_cpu_online()
623 target = cpumask_any_and(&rapl_cpu_mask, get_rapl_pmu_cpumask(cpu)); in rapl_cpu_online()
627 cpumask_set_cpu(cpu, &rapl_cpu_mask); in rapl_cpu_online()
628 pmu->cpu = cpu; in rapl_cpu_online()
638 if (rdmsrl_safe(rm->msr_power_unit, &msr_rapl_power_unit_bits)) in rapl_check_hw_unit()
639 return -1; in rapl_check_hw_unit()
643 switch (rm->unit_quirk) { in rapl_check_hw_unit()
647 * "Intel Xeon Processor E5-1600 and E5-2600 v3 Product Families, V2 in rapl_check_hw_unit()
648 * of 2. Datasheet, September 2014, Reference Number: 330784-001 " in rapl_check_hw_unit()
672 rapl_timer_ms *= (1ULL << (32 - rapl_hw_unit[0] - 1)); in rapl_check_hw_unit()
681 pr_info("API unit is 2^-32 Joules, %d fixed counters, %llu ms ovfl timer\n", in rapl_advertise()
686 pr_info("hw unit of domain %s 2^-%d Joules\n", in rapl_advertise()
696 for (i = 0; i < rapl_pmus->nr_rapl_pmu; i++) in cleanup_rapl_pmus()
697 kfree(rapl_pmus->pmus[i]); in cleanup_rapl_pmus()
719 return -ENOMEM; in init_rapl_pmus()
721 rapl_pmus->nr_rapl_pmu = nr_rapl_pmu; in init_rapl_pmus()
722 rapl_pmus->pmu.attr_groups = rapl_attr_groups; in init_rapl_pmus()
723 rapl_pmus->pmu.attr_update = rapl_attr_update; in init_rapl_pmus()
724 rapl_pmus->pmu.task_ctx_nr = perf_invalid_context; in init_rapl_pmus()
725 rapl_pmus->pmu.event_init = rapl_pmu_event_init; in init_rapl_pmus()
726 rapl_pmus->pmu.add = rapl_pmu_event_add; in init_rapl_pmus()
727 rapl_pmus->pmu.del = rapl_pmu_event_del; in init_rapl_pmus()
728 rapl_pmus->pmu.start = rapl_pmu_event_start; in init_rapl_pmus()
729 rapl_pmus->pmu.stop = rapl_pmu_event_stop; in init_rapl_pmus()
730 rapl_pmus->pmu.read = rapl_pmu_event_read; in init_rapl_pmus()
731 rapl_pmus->pmu.module = THIS_MODULE; in init_rapl_pmus()
732 rapl_pmus->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE; in init_rapl_pmus()
862 return -ENODEV; in rapl_pmu_init()
864 rm = (struct rapl_model *) id->driver_data; in rapl_pmu_init()
866 rapl_msrs = rm->rapl_msrs; in rapl_pmu_init()
869 false, (void *) &rm->events); in rapl_pmu_init()
880 * Install callbacks. Core will call them for each online cpu. in rapl_pmu_init()
888 ret = perf_pmu_register(&rapl_pmus->pmu, "power", -1); in rapl_pmu_init()
907 perf_pmu_unregister(&rapl_pmus->pmu); in intel_rapl_exit()