Lines Matching +full:max +full:- +full:sample +full:- +full:rate +full:- +full:hz
1 // SPDX-License-Identifier: GPL-2.0
3 * Performance event support for the System z CPU-measurement Sampling Facility
29 #define PERF_EVENT_CPUM_SF 0xB0000UL /* Event: Basic-sampling */
30 #define PERF_EVENT_CPUM_SF_DIAG 0xBD000UL /* Event: Combined-sampling */
31 #define PERF_CPUM_SF_BASIC_MODE 0x0001 /* Basic-sampling flag */
32 #define PERF_CPUM_SF_DIAG_MODE 0x0002 /* Diagnostic-sampling flag */
35 #define OVERFLOW_REG(hwc) ((hwc)->extra_reg.config)
36 #define SFB_ALLOC_REG(hwc) ((hwc)->extra_reg.alloc)
37 #define TEAR_REG(hwc) ((hwc)->last_tag)
38 #define SAMPL_RATE(hwc) ((hwc)->event_base)
39 #define SAMPL_FLAGS(hwc) ((hwc)->config_base)
43 /* Minimum number of sample-data-block-tables:
45 * A single table contains up to 511 pointers to sample-data-blocks.
49 /* Number of sample-data-blocks per sample-data-block-table (SDBT):
50 * A table contains SDB pointers (8 bytes) and one table-link entry
53 #define CPUM_SF_SDB_PER_TABLE ((PAGE_SIZE - 8) / 8)
55 /* Maximum page offset for an SDBT table-link entry:
56 * If this page offset is reached, a table-link entry to the next SDBT
68 * the number of sample-data-block-tables into account. Note that these
69 * numbers apply to the basic-sampling function only.
71 * the diagnostic-sampling function is active.
74 * ---------------------------------------------------
76 * 1 page for SDB-tables
80 * 16 pages for SDB-tables
88 unsigned long *sdbt; /* Sample-data-block-table origin */
90 unsigned long num_sdb; /* Number of sample-data-blocks */
91 unsigned long num_sdbt; /* Number of sample-data-block-tables */
92 unsigned long *tail; /* last sample-data-block-table */
105 /* CPU-measurement sampling information block */
107 /* CPU-measurement sampling control block */
123 return (USEC_PER_SEC / freq) * qsi->cpu_speed; in freq_to_sample_rate()
127 unsigned long rate) in sample_rate_to_freq() argument
129 return USEC_PER_SEC * qsi->cpu_speed / rate; in sample_rate_to_freq()
132 /* Return pointer to trailer entry of an sample data block */
139 ret -= sizeof(struct hws_trailer_entry); in trailer_entry_ptr()
145 * Return true if the entry in the sample data block table (sdbt)
160 * sf_disable() - Switch off sampling facility
171 * sf_buffer_available() - Check for an allocated sampling buffer
175 return !!cpuhw->sfb.sdbt; in sf_buffer_available()
185 if (!sfb->sdbt) in free_sampling_buffer()
188 sdbt = sfb->sdbt; in free_sampling_buffer()
196 /* Process table-link entries */ in free_sampling_buffer()
203 if (curr == sfb->sdbt) in free_sampling_buffer()
224 /* Allocate and initialize sample-data-block */ in alloc_sample_data_block()
227 return -ENOMEM; in alloc_sample_data_block()
229 te->header.a = 1; in alloc_sample_data_block()
231 /* Link SDB into the sample-data-block-table */ in alloc_sample_data_block()
238 * realloc_sampling_buffer() - extend sampler memory
240 * Allocates new sample-data-blocks and adds them to the specified sampling
246 * Returns zero on success, non-zero otherwise.
254 if (!sfb->sdbt || !sfb->tail) in realloc_sampling_buffer()
255 return -EINVAL; in realloc_sampling_buffer()
257 if (!is_link_entry(sfb->tail)) in realloc_sampling_buffer()
258 return -EINVAL; in realloc_sampling_buffer()
260 /* Append to the existing sampling buffer, overwriting the table-link in realloc_sampling_buffer()
262 * The tail variables always points to the "tail" (last and table-link) in realloc_sampling_buffer()
263 * entry in an SDB-table. in realloc_sampling_buffer()
265 tail = sfb->tail; in realloc_sampling_buffer()
267 /* Do a sanity check whether the table-link entry points to in realloc_sampling_buffer()
270 if (sfb->sdbt != get_next_sdbt(tail)) { in realloc_sampling_buffer()
272 __func__, (unsigned long)sfb->sdbt, in realloc_sampling_buffer()
274 return -EINVAL; in realloc_sampling_buffer()
280 /* Allocate a new SDB-table if it is full. */ in realloc_sampling_buffer()
284 rc = -ENOMEM; in realloc_sampling_buffer()
287 sfb->num_sdbt++; in realloc_sampling_buffer()
294 /* Allocate a new sample-data-block. in realloc_sampling_buffer()
307 sfb->num_sdbt--; in realloc_sampling_buffer()
313 sfb->num_sdb++; in realloc_sampling_buffer()
319 *tail = virt_to_phys(sfb->sdbt) + 1; in realloc_sampling_buffer()
320 sfb->tail = tail; in realloc_sampling_buffer()
326 * allocate_sampling_buffer() - allocate sampler memory
329 * specified number of sample-data-blocks (SDB). For each allocation,
330 * a 4K page is used. The number of sample-data-block-tables (SDBT)
334 * Returns zero on success, non-zero otherwise.
340 if (sfb->sdbt) in alloc_sampling_buffer()
341 return -EINVAL; in alloc_sampling_buffer()
343 /* Allocate the sample-data-block-table origin */ in alloc_sampling_buffer()
344 sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL); in alloc_sampling_buffer()
345 if (!sfb->sdbt) in alloc_sampling_buffer()
346 return -ENOMEM; in alloc_sampling_buffer()
347 sfb->num_sdb = 0; in alloc_sampling_buffer()
348 sfb->num_sdbt = 1; in alloc_sampling_buffer()
353 sfb->tail = sfb->sdbt; in alloc_sampling_buffer()
354 *sfb->tail = virt_to_phys((void *)sfb->sdbt) + 1; in alloc_sampling_buffer()
356 /* Allocate requested number of sample-data-blocks */ in alloc_sampling_buffer()
363 static void sfb_set_limits(unsigned long min, unsigned long max) in sfb_set_limits() argument
368 CPUM_SF_MAX_SDB = max; in sfb_set_limits()
384 if (!sfb->sdbt) in sfb_pending_allocs()
386 if (SFB_ALLOC_REG(hwc) > sfb->num_sdb) in sfb_pending_allocs()
387 return SFB_ALLOC_REG(hwc) - sfb->num_sdb; in sfb_pending_allocs()
394 num = min_t(unsigned long, num, sfb_max_limit(hwc) - SFB_ALLOC_REG(hwc)); in sfb_account_allocs()
407 if (cpuhw->sfb.sdbt) in deallocate_buffers()
408 free_sampling_buffer(&cpuhw->sfb); in deallocate_buffers()
424 * Control indicator to trigger a measurement-alert to harvest in allocate_buffers()
425 * sample-data-blocks (SDB). This is done per SDB. This in allocate_buffers()
428 * be 2 to 3 SBDs available for sample processing. in allocate_buffers()
430 * n-th page. This is counterproductive as one IRQ triggers in allocate_buffers()
436 * SDBs to handle a higher sampling rate. in allocate_buffers()
438 * (one SDB) for every 10000 HZ frequency increment. in allocate_buffers()
440 * 4. Compute the number of sample-data-block-tables (SDBT) and in allocate_buffers()
444 freq = sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)); in allocate_buffers()
459 return alloc_sampling_buffer(&cpuhw->sfb, in allocate_buffers()
460 sfb_pending_allocs(&cpuhw->sfb, hwc)); in allocate_buffers()
471 /* Use a percentage-based approach to extend the sampling facility in compute_sfb_extent()
472 * buffer. Accept up to 5% sample data loss. in compute_sfb_extent()
474 * sample-data-blocks. in compute_sfb_extent()
500 /* The sample_overflow contains the average number of sample data in sfb_account_overflows()
501 * that has been lost because sample-data-blocks were full. in sfb_account_overflows()
503 * Calculate the total number of sample data entries that has been in sfb_account_overflows()
507 ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb, in sfb_account_overflows()
508 sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc))); in sfb_account_overflows()
510 /* Compute number of sample-data-blocks */ in sfb_account_overflows()
511 num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb); in sfb_account_overflows()
518 /* extend_sampling_buffer() - Extend sampling buffer
546 * new SDB-request cannot be satisfied immediately. in extend_sampling_buffer()
565 qsi(&cpuhw->qsi); in setup_pmc_cpu()
566 cpuhw->flags |= PMU_F_RESERVED; in setup_pmc_cpu()
570 cpuhw->flags &= ~PMU_F_RESERVED; in setup_pmc_cpu()
604 hwc->sample_period = period; in hw_init_period()
605 hwc->last_period = hwc->sample_period; in hw_init_period()
606 local64_set(&hwc->period_left, hwc->sample_period); in hw_init_period()
610 unsigned long rate) in hw_limit_rate() argument
612 return clamp_t(unsigned long, rate, in hw_limit_rate()
613 si->min_sampl_rate, si->max_sampl_rate); in hw_limit_rate()
626 pid = -1; in cpumsf_pid_type()
632 if (event->parent) in cpumsf_pid_type()
633 event = event->parent; in cpumsf_pid_type()
634 pid = __task_pid_nr_ns(tsk, type, event->ns); in cpumsf_pid_type()
640 pid = -1; in cpumsf_pid_type()
655 * Obtain the PID from the basic-sampling data entry and in cpumsf_output_event_pid()
656 * correct the data->tid_entry.pid value. in cpumsf_output_event_pid()
658 pid = data->tid_entry.pid; in cpumsf_output_event_pid()
669 data->tid_entry.pid = cpumsf_pid_type(event, pid, PIDTYPE_TGID); in cpumsf_output_event_pid()
670 data->tid_entry.tid = cpumsf_pid_type(event, pid, PIDTYPE_PID); in cpumsf_output_event_pid()
678 static unsigned long getrate(bool freq, unsigned long sample, in getrate() argument
681 unsigned long rate; in getrate() local
684 rate = freq_to_sample_rate(si, sample); in getrate()
685 rate = hw_limit_rate(si, rate); in getrate()
687 /* The min/max sampling rates specifies the valid range in getrate()
688 * of sample periods. If the specified sample period is in getrate()
691 rate = hw_limit_rate(si, sample); in getrate()
693 /* The perf core maintains a maximum sample rate that is in getrate()
695 * sampling rate does not exceed this value. This also helps in getrate()
699 if (sample_rate_to_freq(si, rate) > in getrate()
701 rate = 0; in getrate()
704 return rate; in getrate()
708 * min/max sampling intervals and the CPU speed. So calculate the
718 * set to the correct sampling rate.
727 struct perf_event_attr *attr = &event->attr; in __hw_perf_event_init_rate()
728 struct hw_perf_event *hwc = &event->hw; in __hw_perf_event_init_rate()
729 unsigned long rate; in __hw_perf_event_init_rate() local
731 if (attr->freq) { in __hw_perf_event_init_rate()
732 if (!attr->sample_freq) in __hw_perf_event_init_rate()
733 return -EINVAL; in __hw_perf_event_init_rate()
734 rate = getrate(attr->freq, attr->sample_freq, si); in __hw_perf_event_init_rate()
735 attr->freq = 0; /* Don't call perf_adjust_period() */ in __hw_perf_event_init_rate()
738 rate = getrate(attr->freq, attr->sample_period, si); in __hw_perf_event_init_rate()
739 if (!rate) in __hw_perf_event_init_rate()
740 return -EINVAL; in __hw_perf_event_init_rate()
742 attr->sample_period = rate; in __hw_perf_event_init_rate()
743 SAMPL_RATE(hwc) = rate; in __hw_perf_event_init_rate()
752 struct perf_event_attr *attr = &event->attr; in __hw_perf_event_init()
753 struct hw_perf_event *hwc = &event->hw; in __hw_perf_event_init()
756 /* Reserve CPU-measurement sampling facility */ in __hw_perf_event_init()
763 event->destroy = hw_perf_event_destroy; in __hw_perf_event_init()
765 /* Access per-CPU sampling information (query sampling info) */ in __hw_perf_event_init()
767 * The event->cpu value can be -1 to count on every CPU, for example, in __hw_perf_event_init()
769 * sampling info from the current CPU, otherwise use event->cpu to in __hw_perf_event_init()
770 * retrieve the per-CPU information. in __hw_perf_event_init()
776 if (event->cpu == -1) { in __hw_perf_event_init()
779 /* Event is pinned to a particular CPU, retrieve the per-CPU in __hw_perf_event_init()
780 * sampling structure for accessing the CPU-specific QSI. in __hw_perf_event_init()
782 cpuhw = &per_cpu(cpu_hw_sf, event->cpu); in __hw_perf_event_init()
783 si = cpuhw->qsi; in __hw_perf_event_init()
791 err = -ENOENT; in __hw_perf_event_init()
797 err = -EBUSY; in __hw_perf_event_init()
807 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) { in __hw_perf_event_init()
809 err = -EPERM; in __hw_perf_event_init()
820 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) in __hw_perf_event_init()
823 /* Allocate the per-CPU sampling buffer using the CPU information in __hw_perf_event_init()
825 * CPU (event->cpu == -1; or cpuhw == NULL), allocate sampling in __hw_perf_event_init()
844 * handler to extract and resolve the PIDs from the basic-sampling in __hw_perf_event_init()
847 if (event->attr.sample_type & PERF_SAMPLE_TID) in __hw_perf_event_init()
849 event->overflow_handler = cpumsf_output_event_pid; in __hw_perf_event_init()
856 u64 sample_type = event->attr.sample_type; in is_callchain_event()
869 return -EOPNOTSUPP; in cpumsf_pmu_event_init()
871 switch (event->attr.type) { in cpumsf_pmu_event_init()
873 if ((event->attr.config != PERF_EVENT_CPUM_SF) && in cpumsf_pmu_event_init()
874 (event->attr.config != PERF_EVENT_CPUM_SF_DIAG)) in cpumsf_pmu_event_init()
875 return -ENOENT; in cpumsf_pmu_event_init()
883 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES) in cpumsf_pmu_event_init()
884 return -ENOENT; in cpumsf_pmu_event_init()
886 return -ENOENT; in cpumsf_pmu_event_init()
889 return -ENOENT; in cpumsf_pmu_event_init()
895 if (event->attr.exclude_hv) in cpumsf_pmu_event_init()
896 event->attr.exclude_hv = 0; in cpumsf_pmu_event_init()
897 if (event->attr.exclude_idle) in cpumsf_pmu_event_init()
898 event->attr.exclude_idle = 0; in cpumsf_pmu_event_init()
902 if (event->destroy) in cpumsf_pmu_event_init()
903 event->destroy(event); in cpumsf_pmu_event_init()
913 if (cpuhw->flags & PMU_F_ENABLED) in cpumsf_pmu_enable()
916 if (cpuhw->flags & PMU_F_ERR_MASK) in cpumsf_pmu_enable()
927 * facility, but it can be fully re-enabled using sampling controls that in cpumsf_pmu_enable()
930 if (cpuhw->event) { in cpumsf_pmu_enable()
931 hwc = &cpuhw->event->hw; in cpumsf_pmu_enable()
934 * Account number of overflow-designated in cpumsf_pmu_enable()
938 extend_sampling_buffer(&cpuhw->sfb, hwc); in cpumsf_pmu_enable()
940 /* Rate may be adjusted with ioctl() */ in cpumsf_pmu_enable()
941 cpuhw->lsctl.interval = SAMPL_RATE(hwc); in cpumsf_pmu_enable()
945 cpuhw->flags |= PMU_F_ENABLED; in cpumsf_pmu_enable()
948 err = lsctl(&cpuhw->lsctl); in cpumsf_pmu_enable()
950 cpuhw->flags &= ~PMU_F_ENABLED; in cpumsf_pmu_enable()
956 lpp(&get_lowcore()->lpp); in cpumsf_pmu_enable()
966 if (!(cpuhw->flags & PMU_F_ENABLED)) in cpumsf_pmu_disable()
969 if (cpuhw->flags & PMU_F_ERR_MASK) in cpumsf_pmu_disable()
973 inactive = cpuhw->lsctl; in cpumsf_pmu_disable()
992 cpuhw->lsctl.tear = si.tear; in cpumsf_pmu_disable()
993 cpuhw->lsctl.dear = si.dear; in cpumsf_pmu_disable()
996 cpuhw->flags &= ~PMU_F_ENABLED; in cpumsf_pmu_disable()
999 /* perf_exclude_event() - Filter event
1002 * @sde_regs: Sample-data-entry (sde) regs structure
1006 * Return non-zero if the event shall be excluded.
1011 if (event->attr.exclude_user && user_mode(regs)) in perf_exclude_event()
1013 if (event->attr.exclude_kernel && !user_mode(regs)) in perf_exclude_event()
1015 if (event->attr.exclude_guest && sde_regs->in_guest) in perf_exclude_event()
1017 if (event->attr.exclude_host && !sde_regs->in_guest) in perf_exclude_event()
1022 /* perf_push_sample() - Push samples to perf
1024 * @sample: Hardware sample data
1026 * Use the hardware sample data to create perf event sample. The sample
1031 * Return non-zero if an event overflow occurred.
1041 /* Setup perf sample */ in perf_push_sample()
1042 perf_sample_data_init(&data, 0, event->hw.last_period); in perf_push_sample()
1044 /* Setup pt_regs to look like an CPU-measurement external interrupt in perf_push_sample()
1046 * field which is unused contains additional sample-data-entry related in perf_push_sample()
1054 psw_bits(regs.psw).ia = basic->ia; in perf_push_sample()
1055 psw_bits(regs.psw).dat = basic->T; in perf_push_sample()
1056 psw_bits(regs.psw).wait = basic->W; in perf_push_sample()
1057 psw_bits(regs.psw).pstate = basic->P; in perf_push_sample()
1058 psw_bits(regs.psw).as = basic->AS; in perf_push_sample()
1062 * sample belongs to a guest or host. If that is not available, in perf_push_sample()
1064 * A non-zero guest program parameter always indicates a guest in perf_push_sample()
1065 * sample. Some early samples or samples from guests without in perf_push_sample()
1071 switch (basic->CL) { in perf_push_sample()
1073 sde_regs->in_guest = 0; in perf_push_sample()
1076 sde_regs->in_guest = 1; in perf_push_sample()
1079 if (basic->gpp || basic->prim_asn != 0xffff) in perf_push_sample()
1080 sde_regs->in_guest = 1; in perf_push_sample()
1085 * Store the PID value from the sample-data-entry to be in perf_push_sample()
1088 data.tid_entry.pid = basic->hpp & LPP_PID_MASK; in perf_push_sample()
1095 event->pmu->stop(event, 0); in perf_push_sample()
1104 local64_add(count, &event->count); in perf_event_count_update()
1107 /* hw_collect_samples() - Walk through a sample-data-block and collect samples
1109 * @sdbt: Sample-data-block table
1112 * Walks through a sample-data-block and collects sampling data entries that are
1114 * there can be either basic-sampling or combined-sampling data entries. A
1115 * combined-sampling data entry consists of a basic- and a diagnostic-sampling
1117 * event hardware structure. The function always works with a combined-sampling
1120 * Note that the implementation focuses on basic-sampling data entries and, if
1121 * such an entry is not valid, the entire combined-sampling data entry is
1131 struct hws_basic_entry *sample; in hw_collect_samples() local
1134 sample = (struct hws_basic_entry *)sdbt; in hw_collect_samples()
1135 while ((unsigned long *)sample < (unsigned long *)te) { in hw_collect_samples()
1136 /* Check for an empty sample */ in hw_collect_samples()
1137 if (!sample->def || sample->LS) in hw_collect_samples()
1141 perf_event_count_update(event, SAMPL_RATE(&event->hw)); in hw_collect_samples()
1143 /* Check whether sample is valid */ in hw_collect_samples()
1144 if (sample->def == 0x0001) { in hw_collect_samples()
1146 * throttle event delivery. Remaining sample data is in hw_collect_samples()
1150 /* Check whether sample is consistent */ in hw_collect_samples()
1151 if (sample->I == 0 && sample->W == 0) { in hw_collect_samples()
1152 /* Deliver sample data to perf */ in hw_collect_samples()
1154 sample); in hw_collect_samples()
1160 /* Sample slot is not yet written or other record. in hw_collect_samples()
1163 * from a combined basic- and diagnostic-sampling. in hw_collect_samples()
1164 * If only basic-sampling is then active, entries are in hw_collect_samples()
1166 * This is typically the case for sample-data-blocks in hw_collect_samples()
1170 if (!te->header.f) in hw_collect_samples()
1174 /* Reset sample slot and advance to next sample */ in hw_collect_samples()
1175 sample->def = 0; in hw_collect_samples()
1176 sample++; in hw_collect_samples()
1180 /* hw_perf_event_update() - Process sampling buffer
1182 * @flush_all: Flag to also flush partially filled sample-data-blocks
1188 * Only full sample-data-blocks are processed. Specify the flush_all flag
1189 * to also walk through partially filled sample-data-blocks.
1195 struct hw_perf_event *hwc = &event->hw; in hw_perf_event_update()
1210 /* Get the trailer entry of the sample-data-block */ in hw_perf_event_update()
1215 if (!te->header.f) { in hw_perf_event_update()
1221 /* Check the sample overflow count */ in hw_perf_event_update()
1222 if (te->header.overflow) in hw_perf_event_update()
1223 /* Account sample overflows and, if a particular limit in hw_perf_event_update()
1227 sampl_overflow += te->header.overflow; in hw_perf_event_update()
1229 /* Collect all samples from a single sample-data-block and in hw_perf_event_update()
1236 /* Reset trailer (using compare-double-and-swap) */ in hw_perf_event_update()
1237 prev.val = READ_ONCE_ALIGNED_128(te->header.val); in hw_perf_event_update()
1244 prev.val = cmpxchg128(&te->header.val, old.val, new.val); in hw_perf_event_update()
1247 /* Advance to next sample-data-block */ in hw_perf_event_update()
1255 /* Stop processing sample-data if all samples of the current in hw_perf_event_update()
1256 * sample-data-block were flushed even if it was not full. in hw_perf_event_update()
1262 /* Account sample overflows in the event hardware structure */ in hw_perf_event_update()
1268 * the interrupt rate to an upper limit. Roughly 1000 samples per in hw_perf_event_update()
1282 return i % aux->sfb.num_sdb; in aux_sdb_index()
1287 return end >= start ? end - start + 1 : 0; in aux_sdb_num()
1292 return aux_sdb_num(aux->head, aux->alert_mark); in aux_sdb_num_alert()
1297 return aux_sdb_num(aux->head, aux->empty_mark); in aux_sdb_num_empty()
1309 sdb = aux->sdb_index[index]; in aux_sdb_trailer()
1333 for (i = 0, idx = aux->head; i < range_scan; i++, idx++) { in aux_output_end()
1335 if (!te->header.f) in aux_output_end()
1342 te = aux_sdb_trailer(aux, aux->alert_mark); in aux_output_end()
1343 te->header.a = 0; in aux_output_end()
1362 if (handle->head & ~PAGE_MASK) in aux_output_begin()
1363 return -EINVAL; in aux_output_begin()
1365 aux->head = handle->head >> PAGE_SHIFT; in aux_output_begin()
1366 range = (handle->size + 1) >> PAGE_SHIFT; in aux_output_begin()
1368 return -ENOMEM; in aux_output_begin()
1371 * SDBs between aux->head and aux->empty_mark are already ready in aux_output_begin()
1375 range_scan = range - aux_sdb_num_empty(aux); in aux_output_begin()
1376 idx = aux->empty_mark + 1; in aux_output_begin()
1379 te->header.f = 0; in aux_output_begin()
1380 te->header.a = 0; in aux_output_begin()
1381 te->header.overflow = 0; in aux_output_begin()
1384 aux->empty_mark = aux->head + range - 1; in aux_output_begin()
1388 aux->alert_mark = aux->head + range/2 - 1; in aux_output_begin()
1389 te = aux_sdb_trailer(aux, aux->alert_mark); in aux_output_begin()
1390 te->header.a = 1; in aux_output_begin()
1393 head = aux_sdb_index(aux, aux->head); in aux_output_begin()
1394 base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE]; in aux_output_begin()
1396 cpuhw->lsctl.tear = virt_to_phys((void *)base) + offset * sizeof(unsigned long); in aux_output_begin()
1397 cpuhw->lsctl.dear = virt_to_phys((void *)aux->sdb_index[head]); in aux_output_begin()
1415 prev.val = READ_ONCE_ALIGNED_128(te->header.val); in aux_set_alert()
1430 prev.val = cmpxchg128(&te->header.val, old.val, new.val); in aux_set_alert()
1436 * aux_reset_buffer() - Scan and setup SDBs for new samples
1438 * @range: The range of SDBs to scan started from aux->head
1441 * Set alert indicator on the SDB at index of aux->alert_mark. If this SDB is
1446 * Scan the SDBs in AUX buffer from behind aux->empty_mark. They are used
1449 * If aux->alert_mark fall in this area, just set it. Overflow count is
1452 * SDBs between aux->head and aux->empty_mark are already reset at last time.
1471 return aux_set_alert(aux, aux->alert_mark, overflow); in aux_reset_buffer()
1473 if (aux->alert_mark <= aux->empty_mark) in aux_reset_buffer()
1478 if (!aux_set_alert(aux, aux->alert_mark, overflow)) in aux_reset_buffer()
1486 range_scan = range - aux_sdb_num_empty(aux); in aux_reset_buffer()
1487 idx = aux->empty_mark + 1; in aux_reset_buffer()
1490 prev.val = READ_ONCE_ALIGNED_128(te->header.val); in aux_reset_buffer()
1497 if (idx == aux->alert_mark) in aux_reset_buffer()
1501 prev.val = cmpxchg128(&te->header.val, old.val, new.val); in aux_reset_buffer()
1507 aux->empty_mark = aux->head + range - 1; in aux_reset_buffer()
1521 struct perf_output_handle *handle = &cpuhw->handle; in hw_collect_aux()
1534 num_sdb = aux->sfb.num_sdb; in hw_collect_aux()
1537 aux = perf_aux_output_begin(handle, cpuhw->event); in hw_collect_aux()
1538 if (handle->size == 0) { in hw_collect_aux()
1540 "diagnostic-sampling mode is full\n", in hw_collect_aux()
1548 aux->head = handle->head >> PAGE_SHIFT; in hw_collect_aux()
1549 range = (handle->size + 1) >> PAGE_SHIFT; in hw_collect_aux()
1551 aux->alert_mark = aux->head; in hw_collect_aux()
1553 aux->alert_mark = aux->head + range/2 - 1; in hw_collect_aux()
1561 perf_aux_output_end(&cpuhw->handle, size); in hw_collect_aux()
1562 pr_err("Sample data caused the AUX buffer with %lu " in hw_collect_aux()
1563 "pages to overflow\n", aux->sfb.num_sdb); in hw_collect_aux()
1566 perf_aux_output_end(&cpuhw->handle, size); in hw_collect_aux()
1583 num_sdbt = aux->sfb.num_sdbt; in aux_buffer_free()
1585 free_page(aux->sdbt_index[i]); in aux_buffer_free()
1587 kfree(aux->sdbt_index); in aux_buffer_free()
1588 kfree(aux->sdb_index); in aux_buffer_free()
1599 te->clock_base = 1; in aux_sdb_init()
1600 te->progusage2 = tod_clock_base.tod; in aux_sdb_init()
1604 * aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling
1605 * @event: Event the buffer is setup for, event->cpu == -1 means current
1613 * the task among online cpus when it is a per-thread event.
1644 sfb = &aux->sfb; in aux_buffer_setup()
1648 aux->sdbt_index = kmalloc_array(n_sdbt, sizeof(void *), GFP_KERNEL); in aux_buffer_setup()
1649 if (!aux->sdbt_index) in aux_buffer_setup()
1653 aux->sdb_index = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL); in aux_buffer_setup()
1654 if (!aux->sdb_index) in aux_buffer_setup()
1658 sfb->num_sdbt = 0; in aux_buffer_setup()
1659 sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL); in aux_buffer_setup()
1660 if (!sfb->sdbt) in aux_buffer_setup()
1662 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt; in aux_buffer_setup()
1663 tail = sfb->tail = sfb->sdbt; in aux_buffer_setup()
1674 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new; in aux_buffer_setup()
1681 aux->sdb_index[i] = (unsigned long)pages[i]; in aux_buffer_setup()
1684 sfb->num_sdb = nr_pages; in aux_buffer_setup()
1687 *tail = virt_to_phys(sfb->sdbt) + 1; in aux_buffer_setup()
1688 sfb->tail = tail; in aux_buffer_setup()
1695 aux->empty_mark = sfb->num_sdb - 1; in aux_buffer_setup()
1701 for (i = 0; i < sfb->num_sdbt; i++) in aux_buffer_setup()
1702 free_page(aux->sdbt_index[i]); in aux_buffer_setup()
1703 kfree(aux->sdb_index); in aux_buffer_setup()
1705 kfree(aux->sdbt_index); in aux_buffer_setup()
1714 /* Nothing to do ... updates are interrupt-driven */ in cpumsf_pmu_read()
1719 * Return non-zero on error and zero on passed checks.
1724 unsigned long rate; in cpumsf_pmu_check_period() local
1728 if (event->cpu == -1) { in cpumsf_pmu_check_period()
1731 /* Event is pinned to a particular CPU, retrieve the per-CPU in cpumsf_pmu_check_period()
1732 * sampling structure for accessing the CPU-specific QSI. in cpumsf_pmu_check_period()
1734 struct cpu_hw_sf *cpuhw = &per_cpu(cpu_hw_sf, event->cpu); in cpumsf_pmu_check_period()
1736 si = cpuhw->qsi; in cpumsf_pmu_check_period()
1739 do_freq = !!SAMPL_FREQ_MODE(&event->hw); in cpumsf_pmu_check_period()
1740 rate = getrate(do_freq, value, &si); in cpumsf_pmu_check_period()
1741 if (!rate) in cpumsf_pmu_check_period()
1742 return -EINVAL; in cpumsf_pmu_check_period()
1744 event->attr.sample_period = rate; in cpumsf_pmu_check_period()
1745 SAMPL_RATE(&event->hw) = rate; in cpumsf_pmu_check_period()
1746 hw_init_period(&event->hw, SAMPL_RATE(&event->hw)); in cpumsf_pmu_check_period()
1757 if (!(event->hw.state & PERF_HES_STOPPED)) in cpumsf_pmu_start()
1759 perf_pmu_disable(event->pmu); in cpumsf_pmu_start()
1760 event->hw.state = 0; in cpumsf_pmu_start()
1761 cpuhw->lsctl.cs = 1; in cpumsf_pmu_start()
1762 if (SAMPL_DIAG_MODE(&event->hw)) in cpumsf_pmu_start()
1763 cpuhw->lsctl.cd = 1; in cpumsf_pmu_start()
1764 perf_pmu_enable(event->pmu); in cpumsf_pmu_start()
1774 if (event->hw.state & PERF_HES_STOPPED) in cpumsf_pmu_stop()
1777 perf_pmu_disable(event->pmu); in cpumsf_pmu_stop()
1778 cpuhw->lsctl.cs = 0; in cpumsf_pmu_stop()
1779 cpuhw->lsctl.cd = 0; in cpumsf_pmu_stop()
1780 event->hw.state |= PERF_HES_STOPPED; in cpumsf_pmu_stop()
1782 if ((flags & PERF_EF_UPDATE) && !(event->hw.state & PERF_HES_UPTODATE)) { in cpumsf_pmu_stop()
1784 event->hw.state |= PERF_HES_UPTODATE; in cpumsf_pmu_stop()
1786 perf_pmu_enable(event->pmu); in cpumsf_pmu_stop()
1795 if (cpuhw->flags & PMU_F_IN_USE) in cpumsf_pmu_add()
1796 return -EAGAIN; in cpumsf_pmu_add()
1798 if (!SAMPL_DIAG_MODE(&event->hw) && !cpuhw->sfb.sdbt) in cpumsf_pmu_add()
1799 return -EINVAL; in cpumsf_pmu_add()
1801 perf_pmu_disable(event->pmu); in cpumsf_pmu_add()
1803 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; in cpumsf_pmu_add()
1806 * using the SDB-table start. Reset TEAR_REG event hardware register in cpumsf_pmu_add()
1810 cpuhw->lsctl.s = 0; in cpumsf_pmu_add()
1811 cpuhw->lsctl.h = 1; in cpumsf_pmu_add()
1812 cpuhw->lsctl.interval = SAMPL_RATE(&event->hw); in cpumsf_pmu_add()
1813 if (!SAMPL_DIAG_MODE(&event->hw)) { in cpumsf_pmu_add()
1814 cpuhw->lsctl.tear = virt_to_phys(cpuhw->sfb.sdbt); in cpumsf_pmu_add()
1815 cpuhw->lsctl.dear = *(unsigned long *)cpuhw->sfb.sdbt; in cpumsf_pmu_add()
1816 TEAR_REG(&event->hw) = (unsigned long)cpuhw->sfb.sdbt; in cpumsf_pmu_add()
1821 if (WARN_ON_ONCE(cpuhw->lsctl.es == 1 || cpuhw->lsctl.ed == 1)) { in cpumsf_pmu_add()
1822 err = -EAGAIN; in cpumsf_pmu_add()
1825 if (SAMPL_DIAG_MODE(&event->hw)) { in cpumsf_pmu_add()
1826 aux = perf_aux_output_begin(&cpuhw->handle, event); in cpumsf_pmu_add()
1828 err = -EINVAL; in cpumsf_pmu_add()
1831 err = aux_output_begin(&cpuhw->handle, aux, cpuhw); in cpumsf_pmu_add()
1834 cpuhw->lsctl.ed = 1; in cpumsf_pmu_add()
1836 cpuhw->lsctl.es = 1; in cpumsf_pmu_add()
1839 cpuhw->event = event; in cpumsf_pmu_add()
1840 cpuhw->flags |= PMU_F_IN_USE; in cpumsf_pmu_add()
1846 perf_pmu_enable(event->pmu); in cpumsf_pmu_add()
1854 perf_pmu_disable(event->pmu); in cpumsf_pmu_del()
1857 cpuhw->lsctl.es = 0; in cpumsf_pmu_del()
1858 cpuhw->lsctl.ed = 0; in cpumsf_pmu_del()
1859 cpuhw->flags &= ~PMU_F_IN_USE; in cpumsf_pmu_del()
1860 cpuhw->event = NULL; in cpumsf_pmu_del()
1862 if (SAMPL_DIAG_MODE(&event->hw)) in cpumsf_pmu_del()
1863 aux_output_end(&cpuhw->handle); in cpumsf_pmu_del()
1865 perf_pmu_enable(event->pmu); in cpumsf_pmu_del()
1897 PMU_FORMAT_ATTR(event, "config:0-63");
1952 if (!(cpuhw->flags & PMU_F_RESERVED)) in cpumf_measurement_alert()
1960 if (cpuhw->flags & PMU_F_IN_USE) in cpumf_measurement_alert()
1961 if (SAMPL_DIAG_MODE(&cpuhw->event->hw)) in cpumf_measurement_alert()
1964 hw_perf_event_update(cpuhw->event, 0); in cpumf_measurement_alert()
1966 WARN_ON_ONCE(!(cpuhw->flags & PMU_F_IN_USE)); in cpumf_measurement_alert()
1969 /* Report measurement alerts only for non-PRA codes */ in cpumf_measurement_alert()
1976 qsi(&cpuhw->qsi); in cpumf_measurement_alert()
1978 /* Loss of sample data due to high-priority machine activities */ in cpumf_measurement_alert()
1980 pr_err("Sample data was lost\n"); in cpumf_measurement_alert()
1981 cpuhw->flags |= PMU_F_ERR_LSDA; in cpumf_measurement_alert()
1989 cpuhw->flags |= PMU_F_ERR_IBE; in cpumf_measurement_alert()
2021 return -ENODEV; in param_get_sfb_size()
2028 unsigned long min, max; in param_set_sfb_size() local
2031 return -ENODEV; in param_set_sfb_size()
2033 return -EINVAL; in param_set_sfb_size()
2035 /* Valid parameter values: "min,max" or "max" */ in param_set_sfb_size()
2037 max = CPUM_SF_MAX_SDB; in param_set_sfb_size()
2039 rc = (sscanf(val, "%lu,%lu", &min, &max) == 2) ? 0 : -EINVAL; in param_set_sfb_size()
2041 rc = kstrtoul(val, 10, &max); in param_set_sfb_size()
2043 if (min < 2 || min >= max || max > get_num_physpages()) in param_set_sfb_size()
2044 rc = -EINVAL; in param_set_sfb_size()
2048 sfb_set_limits(min, max); in param_set_sfb_size()
2050 "min %lu max %lu (diag %lu)\n", in param_set_sfb_size()
2079 return -ENODEV; in init_cpum_sampling_pmu()
2084 return -ENODEV; in init_cpum_sampling_pmu()
2088 return -EINVAL; in init_cpum_sampling_pmu()
2103 return -ENOMEM; in init_cpum_sampling_pmu()