Lines Matching +full:delta +full:- +full:y +full:- +full:threshold
1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
12 * Copyright (C) 2004-2006 Ingo Molnar
60 * A selftest will lurk into the ring-buffer to count the
62 * insertions into the ring-buffer such as trace_printk could occurred
68 * If boot-time tracing including tracers/events via kernel cmdline
120 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
223 strscpy(ftrace_dump_on_oops + 1, str, MAX_TRACER_SIZE - 1); in set_ftrace_dump_on_oops()
247 int left = sizeof(boot_snapshot_info) - boot_snapshot_index; in boot_alloc_snapshot()
253 return -1; in boot_alloc_snapshot()
279 int left = sizeof(boot_instance_info) - boot_instance_index; in boot_instance()
283 return -1; in boot_instance()
346 if (export->flags & flag) { in trace_process_export()
349 export->write(export, entry, size); in trace_process_export()
363 if (export->flags & TRACE_EXPORT_FUNCTION) in ftrace_exports_enable()
366 if (export->flags & TRACE_EXPORT_EVENT) in ftrace_exports_enable()
369 if (export->flags & TRACE_EXPORT_MARKER) in ftrace_exports_enable()
375 if (export->flags & TRACE_EXPORT_FUNCTION) in ftrace_exports_disable()
378 if (export->flags & TRACE_EXPORT_EVENT) in ftrace_exports_disable()
381 if (export->flags & TRACE_EXPORT_MARKER) in ftrace_exports_disable()
394 export = rcu_dereference_raw_check(export->next); in ftrace_exports()
403 rcu_assign_pointer(export->next, *list); in add_trace_export()
407 * the export->next pointer is valid before another CPU sees in add_trace_export()
418 for (p = list; *p != NULL; p = &(*p)->next) in rm_trace_export()
423 return -1; in rm_trace_export()
425 rcu_assign_pointer(*p, (*p)->next); in rm_trace_export()
451 if (WARN_ON_ONCE(!export->write)) in register_ftrace_export()
452 return -1; in register_ftrace_export()
496 * The global_trace is the descriptor that holds the top-level tracing
513 return !(tr->flags & TRACE_ARRAY_FL_BOOT); in printk_binsafe()
521 printk_trace->trace_flags &= ~TRACE_ITER_TRACE_PRINTK; in update_printk_trace()
523 tr->trace_flags |= TRACE_ITER_TRACE_PRINTK; in update_printk_trace()
530 tr->ring_buffer_expanded = true; in trace_set_ring_buffer_expanded()
538 int ret = -ENODEV; in trace_array_get()
543 tr->ref++; in trace_array_get()
555 WARN_ON(!this_tr->ref); in __trace_array_put()
556 this_tr->ref--; in __trace_array_put()
560 * trace_array_put - Decrement the reference counter for this trace array.
588 return -ENODEV; in tracing_check_open_get_tr()
591 return -ENODEV; in tracing_check_open_get_tr()
600 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) && in call_filter_check_discard()
601 !filter_match_preds(call->filter, rec)) { in call_filter_check_discard()
610 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
623 * trace_ignore_this_task - should a task be ignored for tracing
646 !trace_find_filtered_pid(filtered_pids, task->pid)) || in trace_ignore_this_task()
648 trace_find_filtered_pid(filtered_no_pids, task->pid)); in trace_ignore_this_task()
652 * trace_filter_add_remove_task - Add or remove a task from a pid_list
672 if (!trace_find_filtered_pid(pid_list, self->pid)) in trace_filter_add_remove_task()
678 trace_pid_list_set(pid_list, task->pid); in trace_filter_add_remove_task()
680 trace_pid_list_clear(pid_list, task->pid); in trace_filter_add_remove_task()
684 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
713 * trace_pid_start - Used for seq_file to start reading pid lists
742 * trace_pid_show - show the current pid in seq_file processing
751 unsigned long pid = (unsigned long)v - 1; in trace_pid_show()
774 return -ENOMEM; in trace_pid_write()
785 return -ENOMEM; in trace_pid_write()
809 cnt -= ret; in trace_pid_write()
814 ret = -EINVAL; in trace_pid_write()
821 ret = -1; in trace_pid_write()
852 if (!buf->buffer) in buffer_ftrace_now()
855 ts = ring_buffer_time_stamp(buf->buffer); in buffer_ftrace_now()
856 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts); in buffer_ftrace_now()
867 * tracing_is_enabled - Show if global_trace has been enabled
926 * These primitives don't distinguish read-only and read-consume access.
927 * Multi read-only access are also serialized.
1040 if (tr->array_buffer.buffer) in tracer_tracing_on()
1041 ring_buffer_record_on(tr->array_buffer.buffer); in tracer_tracing_on()
1050 tr->buffer_disabled = 0; in tracer_tracing_on()
1056 * tracing_on - enable tracing buffers
1075 /* Length is in event->array[0] */ in __buffer_unlock_commit()
1076 ring_buffer_write(buffer, event->array[0], &event->array[1]); in __buffer_unlock_commit()
1094 if (!(tr->trace_flags & TRACE_ITER_PRINTK)) in __trace_array_puts()
1106 buffer = tr->array_buffer.buffer; in __trace_array_puts()
1116 entry->ip = ip; in __trace_array_puts()
1118 memcpy(&entry->buf, str, size); in __trace_array_puts()
1121 if (entry->buf[size - 1] != '\n') { in __trace_array_puts()
1122 entry->buf[size] = '\n'; in __trace_array_puts()
1123 entry->buf[size + 1] = '\0'; in __trace_array_puts()
1125 entry->buf[size] = '\0'; in __trace_array_puts()
1136 * __trace_puts - write a constant string into the trace buffer.
1148 * __trace_bputs - write the pointer to a constant string into trace buffer
1165 if (!(tr->trace_flags & TRACE_ITER_PRINTK)) in __trace_bputs()
1172 buffer = tr->array_buffer.buffer; in __trace_bputs()
1181 entry->ip = ip; in __trace_bputs()
1182 entry->str = str; in __trace_bputs()
1198 struct tracer *tracer = tr->current_trace; in tracing_snapshot_instance_cond()
1207 if (!tr->allocated_snapshot) { in tracing_snapshot_instance_cond()
1215 if (tracer->use_max_tr) { in tracing_snapshot_instance_cond()
1221 if (tr->mapped) { in tracing_snapshot_instance_cond()
1238 * tracing_snapshot - take a snapshot of the current buffer.
1260 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
1265 * conditional - the snapshot will only happen if the
1279 * tracing_cond_snapshot_data - get the user data associated with a snapshot
1283 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1287 * the tr->max_lock lock, which the code calling
1297 arch_spin_lock(&tr->max_lock); in tracing_cond_snapshot_data()
1299 if (tr->cond_snapshot) in tracing_cond_snapshot_data()
1300 cond_data = tr->cond_snapshot->cond_data; in tracing_cond_snapshot_data()
1302 arch_spin_unlock(&tr->max_lock); in tracing_cond_snapshot_data()
1318 if (!tr->allocated_snapshot) { in tracing_alloc_snapshot_instance()
1321 order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); in tracing_alloc_snapshot_instance()
1322 ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order); in tracing_alloc_snapshot_instance()
1327 ret = resize_buffer_duplicate_size(&tr->max_buffer, in tracing_alloc_snapshot_instance()
1328 &tr->array_buffer, RING_BUFFER_ALL_CPUS); in tracing_alloc_snapshot_instance()
1332 tr->allocated_snapshot = true; in tracing_alloc_snapshot_instance()
1342 * The max_tr ring buffer has some state (e.g. ring->clock) and in free_snapshot()
1345 ring_buffer_subbuf_order_set(tr->max_buffer.buffer, 0); in free_snapshot()
1346 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); in free_snapshot()
1347 set_buffer_entries(&tr->max_buffer, 1); in free_snapshot()
1348 tracing_reset_online_cpus(&tr->max_buffer); in free_snapshot()
1349 tr->allocated_snapshot = false; in free_snapshot()
1358 spin_lock(&tr->snapshot_trigger_lock); in tracing_arm_snapshot_locked()
1359 if (tr->snapshot == UINT_MAX || tr->mapped) { in tracing_arm_snapshot_locked()
1360 spin_unlock(&tr->snapshot_trigger_lock); in tracing_arm_snapshot_locked()
1361 return -EBUSY; in tracing_arm_snapshot_locked()
1364 tr->snapshot++; in tracing_arm_snapshot_locked()
1365 spin_unlock(&tr->snapshot_trigger_lock); in tracing_arm_snapshot_locked()
1369 spin_lock(&tr->snapshot_trigger_lock); in tracing_arm_snapshot_locked()
1370 tr->snapshot--; in tracing_arm_snapshot_locked()
1371 spin_unlock(&tr->snapshot_trigger_lock); in tracing_arm_snapshot_locked()
1390 spin_lock(&tr->snapshot_trigger_lock); in tracing_disarm_snapshot()
1391 if (!WARN_ON(!tr->snapshot)) in tracing_disarm_snapshot()
1392 tr->snapshot--; in tracing_disarm_snapshot()
1393 spin_unlock(&tr->snapshot_trigger_lock); in tracing_disarm_snapshot()
1397 * tracing_alloc_snapshot - allocate snapshot buffer.
1400 * allocated - it doesn't also take a snapshot.
1419 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1442 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1449 * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1462 return -ENOMEM; in tracing_snapshot_cond_enable()
1464 cond_snapshot->cond_data = cond_data; in tracing_snapshot_cond_enable()
1465 cond_snapshot->update = update; in tracing_snapshot_cond_enable()
1469 if (tr->current_trace->use_max_tr) { in tracing_snapshot_cond_enable()
1470 ret = -EBUSY; in tracing_snapshot_cond_enable()
1482 if (tr->cond_snapshot) { in tracing_snapshot_cond_enable()
1483 ret = -EBUSY; in tracing_snapshot_cond_enable()
1492 arch_spin_lock(&tr->max_lock); in tracing_snapshot_cond_enable()
1493 tr->cond_snapshot = cond_snapshot; in tracing_snapshot_cond_enable()
1494 arch_spin_unlock(&tr->max_lock); in tracing_snapshot_cond_enable()
1509 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1514 * otherwise return -EINVAL.
1523 arch_spin_lock(&tr->max_lock); in tracing_snapshot_cond_disable()
1525 if (!tr->cond_snapshot) in tracing_snapshot_cond_disable()
1526 ret = -EINVAL; in tracing_snapshot_cond_disable()
1528 kfree(tr->cond_snapshot); in tracing_snapshot_cond_disable()
1529 tr->cond_snapshot = NULL; in tracing_snapshot_cond_disable()
1532 arch_spin_unlock(&tr->max_lock); in tracing_snapshot_cond_disable()
1554 return -ENODEV; in tracing_alloc_snapshot()
1570 return -ENODEV; in tracing_snapshot_cond_enable()
1579 #define tracing_arm_snapshot_locked(tr) ({ -EBUSY; })
1584 if (tr->array_buffer.buffer) in tracer_tracing_off()
1585 ring_buffer_record_off(tr->array_buffer.buffer); in tracer_tracing_off()
1594 tr->buffer_disabled = 1; in tracer_tracing_off()
1600 * tracing_off - turn off tracing buffers
1623 * tracer_tracing_is_on - show real state of ring buffer enabled
1630 if (tr->array_buffer.buffer) in tracer_tracing_is_on()
1631 return ring_buffer_record_is_set_on(tr->array_buffer.buffer); in tracer_tracing_is_on()
1632 return !tr->buffer_disabled; in tracer_tracing_is_on()
1636 * tracing_is_on - show state of ring buffers enabled
1663 unsigned long threshold; in set_tracing_thresh() local
1668 ret = kstrtoul(str, 0, &threshold); in set_tracing_thresh()
1671 tracing_thresh = threshold * 1000; in set_tracing_thresh()
1715 if (trace_clocks[tr->clock_id].in_ns) in trace_clock_in_ns()
1722 * trace_parser_get_init - gets the buffer for trace parser
1728 parser->buffer = kmalloc(size, GFP_KERNEL); in trace_parser_get_init()
1729 if (!parser->buffer) in trace_parser_get_init()
1732 parser->size = size; in trace_parser_get_init()
1737 * trace_parser_put - frees the buffer for trace parser
1741 kfree(parser->buffer); in trace_parser_put()
1742 parser->buffer = NULL; in trace_parser_put()
1746 * trace_get_user - reads the user input string separated by space
1771 cnt--; in trace_get_user()
1777 if (!parser->cont) { in trace_get_user()
1784 cnt--; in trace_get_user()
1787 parser->idx = 0; in trace_get_user()
1797 /* read the non-space input */ in trace_get_user()
1799 if (parser->idx < parser->size - 1) in trace_get_user()
1800 parser->buffer[parser->idx++] = ch; in trace_get_user()
1802 ret = -EINVAL; in trace_get_user()
1809 cnt--; in trace_get_user()
1814 parser->buffer[parser->idx] = 0; in trace_get_user()
1815 parser->cont = false; in trace_get_user()
1816 } else if (parser->idx < parser->size - 1) { in trace_get_user()
1817 parser->cont = true; in trace_get_user()
1818 parser->buffer[parser->idx++] = ch; in trace_get_user()
1820 parser->buffer[parser->idx] = 0; in trace_get_user()
1822 ret = -EINVAL; in trace_get_user()
1838 if (trace_seq_used(s) <= s->readpos) in trace_seq_to_buffer()
1839 return -EBUSY; in trace_seq_to_buffer()
1841 len = trace_seq_used(s) - s->readpos; in trace_seq_to_buffer()
1844 memcpy(buf, s->buffer + s->readpos, cnt); in trace_seq_to_buffer()
1846 s->readpos += cnt; in trace_seq_to_buffer()
1863 fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY); in latency_fsnotify_workfn()
1870 queue_work(fsnotify_wq, &tr->fsnotify_work); in latency_fsnotify_workfn_irq()
1876 INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn); in trace_create_maxlat_file()
1877 init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq); in trace_create_maxlat_file()
1878 tr->d_max_latency = trace_create_file("tracing_max_latency", in trace_create_maxlat_file()
1890 return -ENOMEM; in latency_fsnotify_init()
1902 * We cannot call queue_work(&tr->fsnotify_work) from here because it's in latency_fsnotify()
1906 irq_work_queue(&tr->fsnotify_irqwork); in latency_fsnotify()
1918 * Copy the new maximum trace into the separate maximum-trace
1925 struct array_buffer *trace_buf = &tr->array_buffer; in __update_max_tr()
1926 struct array_buffer *max_buf = &tr->max_buffer; in __update_max_tr()
1927 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu); in __update_max_tr()
1928 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu); in __update_max_tr()
1930 max_buf->cpu = cpu; in __update_max_tr()
1931 max_buf->time_start = data->preempt_timestamp; in __update_max_tr()
1933 max_data->saved_latency = tr->max_latency; in __update_max_tr()
1934 max_data->critical_start = data->critical_start; in __update_max_tr()
1935 max_data->critical_end = data->critical_end; in __update_max_tr()
1937 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN); in __update_max_tr()
1938 max_data->pid = tsk->pid; in __update_max_tr()
1944 max_data->uid = current_uid(); in __update_max_tr()
1946 max_data->uid = task_uid(tsk); in __update_max_tr()
1948 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; in __update_max_tr()
1949 max_data->policy = tsk->policy; in __update_max_tr()
1950 max_data->rt_priority = tsk->rt_priority; in __update_max_tr()
1958 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1971 if (tr->stop_count) in update_max_tr()
1976 if (!tr->allocated_snapshot) { in update_max_tr()
1978 WARN_ON_ONCE(tr->current_trace != &nop_trace); in update_max_tr()
1982 arch_spin_lock(&tr->max_lock); in update_max_tr()
1985 if (ring_buffer_record_is_set_on(tr->array_buffer.buffer)) in update_max_tr()
1986 ring_buffer_record_on(tr->max_buffer.buffer); in update_max_tr()
1988 ring_buffer_record_off(tr->max_buffer.buffer); in update_max_tr()
1991 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) { in update_max_tr()
1992 arch_spin_unlock(&tr->max_lock); in update_max_tr()
1996 swap(tr->array_buffer.buffer, tr->max_buffer.buffer); in update_max_tr()
2000 arch_spin_unlock(&tr->max_lock); in update_max_tr()
2003 ring_buffer_wake_waiters(tr->array_buffer.buffer, RING_BUFFER_ALL_CPUS); in update_max_tr()
2007 * update_max_tr_single - only copy one trace over, and reset the rest
2019 if (tr->stop_count) in update_max_tr_single()
2023 if (!tr->allocated_snapshot) { in update_max_tr_single()
2025 WARN_ON_ONCE(tr->current_trace != &nop_trace); in update_max_tr_single()
2029 arch_spin_lock(&tr->max_lock); in update_max_tr_single()
2031 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu); in update_max_tr_single()
2033 if (ret == -EBUSY) { in update_max_tr_single()
2041 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_, in update_max_tr_single()
2045 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); in update_max_tr_single()
2048 arch_spin_unlock(&tr->max_lock); in update_max_tr_single()
2061 struct trace_iterator *iter = pwait->iter; in wait_pipe_cond()
2063 if (atomic_read_acquire(&iter->wait_index) != pwait->wait_index) in wait_pipe_cond()
2066 return iter->closed; in wait_pipe_cond()
2075 if (trace_buffer_iter(iter, iter->cpu_file)) in wait_on_pipe()
2078 pwait.wait_index = atomic_read_acquire(&iter->wait_index); in wait_on_pipe()
2081 ret = ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file, full, in wait_on_pipe()
2089 if (iter->snapshot) in wait_on_pipe()
2090 iter->array_buffer = &iter->tr->max_buffer; in wait_on_pipe()
2111 return -ENOMEM; in save_selftest()
2113 selftest->type = type; in save_selftest()
2114 list_add(&selftest->list, &postponed_selftests); in save_selftest()
2121 struct tracer *saved_tracer = tr->current_trace; in run_tracer_selftest()
2124 if (!type->selftest || tracing_selftest_disabled) in run_tracer_selftest()
2137 type->name); in run_tracer_selftest()
2148 tracing_reset_online_cpus(&tr->array_buffer); in run_tracer_selftest()
2150 tr->current_trace = type; in run_tracer_selftest()
2153 if (type->use_max_tr) { in run_tracer_selftest()
2155 if (tr->ring_buffer_expanded) in run_tracer_selftest()
2156 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size, in run_tracer_selftest()
2158 tr->allocated_snapshot = true; in run_tracer_selftest()
2163 pr_info("Testing tracer %s: ", type->name); in run_tracer_selftest()
2164 ret = type->selftest(type, tr); in run_tracer_selftest()
2166 tr->current_trace = saved_tracer; in run_tracer_selftest()
2171 return -1; in run_tracer_selftest()
2174 tracing_reset_online_cpus(&tr->array_buffer); in run_tracer_selftest()
2177 if (type->use_max_tr) { in run_tracer_selftest()
2178 tr->allocated_snapshot = false; in run_tracer_selftest()
2181 if (tr->ring_buffer_expanded) in run_tracer_selftest()
2182 ring_buffer_resize(tr->max_buffer.buffer, 1, in run_tracer_selftest()
2230 ret = run_tracer_selftest(p->type); in init_trace_selftests()
2234 p->type->name); in init_trace_selftests()
2236 for (t = trace_types; t; t = t->next) { in init_trace_selftests()
2237 if (t == p->type) { in init_trace_selftests()
2238 *last = t->next; in init_trace_selftests()
2241 last = &t->next; in init_trace_selftests()
2244 list_del(&p->list); in init_trace_selftests()
2267 * register_tracer - register a tracer with the ftrace system.
2277 if (!type->name) { in register_tracer()
2279 return -1; in register_tracer()
2282 if (strlen(type->name) >= MAX_TRACER_SIZE) { in register_tracer()
2284 return -1; in register_tracer()
2289 type->name); in register_tracer()
2290 return -EPERM; in register_tracer()
2295 for (t = trace_types; t; t = t->next) { in register_tracer()
2296 if (strcmp(type->name, t->name) == 0) { in register_tracer()
2299 type->name); in register_tracer()
2300 ret = -1; in register_tracer()
2305 if (!type->set_flag) in register_tracer()
2306 type->set_flag = &dummy_set_flag; in register_tracer()
2307 if (!type->flags) { in register_tracer()
2309 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL); in register_tracer()
2310 if (!type->flags) { in register_tracer()
2311 ret = -ENOMEM; in register_tracer()
2314 type->flags->val = 0; in register_tracer()
2315 type->flags->opts = dummy_tracer_opt; in register_tracer()
2317 if (!type->flags->opts) in register_tracer()
2318 type->flags->opts = dummy_tracer_opt; in register_tracer()
2321 type->flags->trace = type; in register_tracer()
2327 type->next = trace_types; in register_tracer()
2337 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE)) in register_tracer()
2340 printk(KERN_INFO "Starting tracer '%s'\n", type->name); in register_tracer()
2342 tracing_set_tracer(&global_trace, type->name); in register_tracer()
2356 struct trace_buffer *buffer = buf->buffer; in tracing_reset_cpu()
2372 struct trace_buffer *buffer = buf->buffer; in tracing_reset_online_cpus()
2382 buf->time_start = buffer_ftrace_now(buf, buf->cpu); in tracing_reset_online_cpus()
2391 struct trace_buffer *buffer = buf->buffer; in tracing_reset_all_cpus()
2401 buf->time_start = buffer_ftrace_now(buf, buf->cpu); in tracing_reset_all_cpus()
2416 if (!tr->clear_trace) in tracing_reset_all_online_cpus_unlocked()
2418 tr->clear_trace = false; in tracing_reset_all_online_cpus_unlocked()
2419 tracing_reset_online_cpus(&tr->array_buffer); in tracing_reset_all_online_cpus_unlocked()
2421 tracing_reset_online_cpus(&tr->max_buffer); in tracing_reset_all_online_cpus_unlocked()
2446 raw_spin_lock_irqsave(&tr->start_lock, flags); in tracing_start_tr()
2447 if (--tr->stop_count) { in tracing_start_tr()
2448 if (WARN_ON_ONCE(tr->stop_count < 0)) { in tracing_start_tr()
2450 tr->stop_count = 0; in tracing_start_tr()
2456 arch_spin_lock(&tr->max_lock); in tracing_start_tr()
2458 buffer = tr->array_buffer.buffer; in tracing_start_tr()
2463 buffer = tr->max_buffer.buffer; in tracing_start_tr()
2468 arch_spin_unlock(&tr->max_lock); in tracing_start_tr()
2471 raw_spin_unlock_irqrestore(&tr->start_lock, flags); in tracing_start_tr()
2475 * tracing_start - quick start of the tracer
2491 raw_spin_lock_irqsave(&tr->start_lock, flags); in tracing_stop_tr()
2492 if (tr->stop_count++) in tracing_stop_tr()
2496 arch_spin_lock(&tr->max_lock); in tracing_stop_tr()
2498 buffer = tr->array_buffer.buffer; in tracing_stop_tr()
2503 buffer = tr->max_buffer.buffer; in tracing_stop_tr()
2508 arch_spin_unlock(&tr->max_lock); in tracing_stop_tr()
2511 raw_spin_unlock_irqrestore(&tr->start_lock, flags); in tracing_stop_tr()
2515 * tracing_stop - quick stop of the tracer
2540 return current->migration_disabled; in migration_disable_value()
2584 * trace_buffered_event_enable - enable buffering events
2644 * trace_buffered_event_disable - disable buffering events
2660 if (--trace_buffered_event_ref) in trace_buffered_event_disable()
2680 * could wrongly decide to use the pointed-to buffer which is now freed. in trace_buffered_event_disable()
2698 struct trace_array *tr = trace_file->tr; in trace_event_buffer_lock_reserve()
2701 *current_rb = tr->array_buffer.buffer; in trace_event_buffer_lock_reserve()
2703 if (!tr->no_filter_buffering_ref && in trace_event_buffer_lock_reserve()
2704 (trace_file->flags & (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED))) { in trace_event_buffer_lock_reserve()
2711 * (see include/linux/ring-buffer.h for details on in trace_event_buffer_lock_reserve()
2724 int max_len = PAGE_SIZE - struct_size(entry, array, 1); in trace_event_buffer_lock_reserve()
2747 entry->array[0] = len; in trace_event_buffer_lock_reserve()
2765 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) { in trace_event_buffer_lock_reserve()
2789 event_call = fbuffer->trace_file->event_call; in output_printk()
2790 if (!event_call || !event_call->event.funcs || in output_printk()
2791 !event_call->event.funcs->trace) in output_printk()
2794 file = fbuffer->trace_file; in output_printk()
2795 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) || in output_printk()
2796 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) && in output_printk()
2797 !filter_match_preds(file->filter, fbuffer->entry))) in output_printk()
2800 event = &fbuffer->trace_file->event_call->event; in output_printk()
2803 trace_seq_init(&iter->seq); in output_printk()
2804 iter->ent = fbuffer->entry; in output_printk()
2805 event_call->event.funcs->trace(iter, 0, event); in output_printk()
2806 trace_seq_putc(&iter->seq, 0); in output_printk()
2807 printk("%s", iter->seq.buffer); in output_printk()
2848 struct trace_event_file *file = fbuffer->trace_file; in trace_event_buffer_commit()
2850 if (__event_trigger_test_discard(file, fbuffer->buffer, fbuffer->event, in trace_event_buffer_commit()
2851 fbuffer->entry, &tt)) in trace_event_buffer_commit()
2858 ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT); in trace_event_buffer_commit()
2860 trace_buffer_unlock_commit_regs(file->tr, fbuffer->buffer, in trace_event_buffer_commit()
2861 fbuffer->event, fbuffer->trace_ctx, fbuffer->regs); in trace_event_buffer_commit()
2912 struct trace_buffer *buffer = tr->array_buffer.buffer; in trace_function()
2921 entry->ip = ip; in trace_function()
2922 entry->parent_ip = parent_ip; in trace_function()
2972 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1; in __ftrace_trace_stack()
2988 size = ARRAY_SIZE(fstack->calls); in __ftrace_trace_stack()
2991 nr_entries = stack_trace_save_regs(regs, fstack->calls, in __ftrace_trace_stack()
2994 nr_entries = stack_trace_save(fstack->calls, size, skip); in __ftrace_trace_stack()
3004 entry->size = nr_entries; in __ftrace_trace_stack()
3005 memcpy(&entry->caller, fstack->calls, in __ftrace_trace_stack()
3024 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE)) in ftrace_trace_stack()
3033 struct trace_buffer *buffer = tr->array_buffer.buffer; in __trace_stack()
3058 * trace_dump_stack - record a stack back trace in the trace buffer
3070 __ftrace_trace_stack(printk_trace->array_buffer.buffer, in trace_dump_stack()
3086 if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE)) in ftrace_trace_userstack()
3112 entry->tgid = current->tgid; in ftrace_trace_userstack()
3113 memset(&entry->caller, 0, sizeof(entry->caller)); in ftrace_trace_userstack()
3115 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES); in ftrace_trace_userstack()
3136 unsigned long long delta) in func_repeats_set_delta_ts() argument
3138 entry->bottom_delta_ts = delta & U32_MAX; in func_repeats_set_delta_ts()
3139 entry->top_delta_ts = (delta >> 32); in func_repeats_set_delta_ts()
3146 struct trace_buffer *buffer = tr->array_buffer.buffer; in trace_last_func_repeats()
3149 u64 delta; in trace_last_func_repeats() local
3156 delta = ring_buffer_event_time_stamp(buffer, event) - in trace_last_func_repeats()
3157 last_info->ts_last_call; in trace_last_func_repeats()
3160 entry->ip = last_info->ip; in trace_last_func_repeats()
3161 entry->parent_ip = last_info->parent_ip; in trace_last_func_repeats()
3162 entry->count = last_info->count; in trace_last_func_repeats()
3163 func_repeats_set_delta_ts(entry, delta); in trace_last_func_repeats()
3184 if (!trace_percpu_buffer || buffer->nesting >= 4) in get_trace_buf()
3187 buffer->nesting++; in get_trace_buf()
3191 return &buffer->buffer[buffer->nesting - 1][0]; in get_trace_buf()
3198 this_cpu_dec(trace_percpu_buffer->nesting); in put_trace_buf()
3210 return -ENOMEM; in alloc_percpu_trace_buffer()
3279 * trace_vbprintk - write binary msg to tracing buffer
3319 buffer = tr->array_buffer.buffer; in trace_vbprintk()
3326 entry->ip = ip; in trace_vbprintk()
3327 entry->fmt = fmt; in trace_vbprintk()
3329 memcpy(entry->buf, tbuffer, sizeof(u32) * len); in trace_vbprintk()
3385 entry->ip = ip; in __trace_array_vprintk()
3387 memcpy(&entry->buf, tbuffer, len + 1); in __trace_array_vprintk()
3411 return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args); in trace_array_vprintk()
3415 * trace_array_printk - Print a message to a specific instance
3442 return -ENOENT; in trace_array_printk()
3448 if (!(tr->trace_flags & TRACE_ITER_PRINTK)) in trace_array_printk()
3459 * trace_array_init_printk - Initialize buffers for trace_array_printk()
3469 return -ENOENT; in trace_array_init_printk()
3473 return -EINVAL; in trace_array_init_printk()
3486 if (!(printk_trace->trace_flags & TRACE_ITER_PRINTK)) in trace_array_printk_buf()
3504 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu); in trace_iterator_increment()
3506 iter->idx++; in trace_iterator_increment()
3522 (unsigned long)-1 : 0; in peek_next_entry()
3524 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts, in peek_next_entry()
3529 iter->ent_size = ring_buffer_event_length(event); in peek_next_entry()
3532 iter->ent_size = 0; in peek_next_entry()
3540 struct trace_buffer *buffer = iter->array_buffer->buffer; in __find_next_entry()
3543 int cpu_file = iter->cpu_file; in __find_next_entry()
3545 int next_cpu = -1; in __find_next_entry()
3578 next_size = iter->ent_size; in __find_next_entry()
3582 iter->ent_size = next_size; in __find_next_entry()
3604 * iter->tr is NULL when used with tp_printk, which makes in trace_iter_expand_format()
3607 if (!iter->tr || iter->fmt == static_fmt_buf) in trace_iter_expand_format()
3610 tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE, in trace_iter_expand_format()
3613 iter->fmt_size += STATIC_FMT_BUF_SIZE; in trace_iter_expand_format()
3614 iter->fmt = tmp; in trace_iter_expand_format()
3633 if ((addr >= (unsigned long)iter->ent) && in trace_safe_str()
3634 (addr < (unsigned long)iter->ent + iter->ent_size)) in trace_safe_str()
3638 if ((addr >= (unsigned long)iter->tmp_seq.buffer) && in trace_safe_str()
3639 (addr < (unsigned long)iter->tmp_seq.buffer + TRACE_SEQ_BUFFER_SIZE)) in trace_safe_str()
3653 if (!iter->ent) in trace_safe_str()
3656 trace_event = ftrace_find_event(iter->ent->type); in trace_safe_str()
3661 if ((event->flags & TRACE_EVENT_FL_DYNAMIC) || !event->module) in trace_safe_str()
3665 if (within_module_core(addr, event->module)) in trace_safe_str()
3705 * trace_check_vprintf - Check dereferenced strings while writing to the seq buffer
3710 * This writes the data into the @iter->seq buffer using the data from
3738 if (iter->tr) { in trace_check_vprintf()
3739 text_delta = iter->tr->text_delta; in trace_check_vprintf()
3740 data_delta = iter->tr->data_delta; in trace_check_vprintf()
3744 if (iter->fmt == static_fmt_buf) in trace_check_vprintf()
3755 * as well as %p[sS] if delta is non-zero in trace_check_vprintf()
3758 if (i + 1 >= iter->fmt_size) { in trace_check_vprintf()
3799 strncpy(iter->fmt, p, i); in trace_check_vprintf()
3800 iter->fmt[i] = '\0'; in trace_check_vprintf()
3801 trace_seq_vprintf(&iter->seq, iter->fmt, ap); in trace_check_vprintf()
3803 /* Add delta to %pS pointers */ in trace_check_vprintf()
3815 trace_seq_printf(&iter->seq, fmt, (void *)addr); in trace_check_vprintf()
3822 * If iter->seq is full, the above call no longer guarantees in trace_check_vprintf()
3828 if (iter->seq.full) { in trace_check_vprintf()
3853 * instead. See samples/trace_events/trace-events-sample.h in trace_check_vprintf()
3857 fmt, seq_buf_str(&iter->seq.seq))) { in trace_check_vprintf()
3862 if (len + 1 > iter->fmt_size) in trace_check_vprintf()
3863 len = iter->fmt_size - 1; in trace_check_vprintf()
3866 ret = copy_from_kernel_nofault(iter->fmt, str, len); in trace_check_vprintf()
3867 iter->fmt[len] = 0; in trace_check_vprintf()
3870 ret = strncpy_from_kernel_nofault(iter->fmt, str, in trace_check_vprintf()
3871 iter->fmt_size); in trace_check_vprintf()
3874 trace_seq_printf(&iter->seq, "(0x%px)", str); in trace_check_vprintf()
3876 trace_seq_printf(&iter->seq, "(0x%px:%s)", in trace_check_vprintf()
3877 str, iter->fmt); in trace_check_vprintf()
3878 str = "[UNSAFE-MEMORY]"; in trace_check_vprintf()
3879 strcpy(iter->fmt, "%s"); in trace_check_vprintf()
3881 strncpy(iter->fmt, p + i, j + 1); in trace_check_vprintf()
3882 iter->fmt[j+1] = '\0'; in trace_check_vprintf()
3885 trace_seq_printf(&iter->seq, iter->fmt, len, str); in trace_check_vprintf()
3887 trace_seq_printf(&iter->seq, iter->fmt, str); in trace_check_vprintf()
3893 trace_seq_vprintf(&iter->seq, p, ap); in trace_check_vprintf()
3904 if (!iter->tr || iter->tr->trace_flags & TRACE_ITER_HASH_PTR) in trace_event_format()
3908 new_fmt = q = iter->fmt; in trace_event_format()
3910 if (unlikely(q - new_fmt + 3 > iter->fmt_size)) { in trace_event_format()
3914 q += iter->fmt - new_fmt; in trace_event_format()
3915 new_fmt = iter->fmt; in trace_event_format()
3921 if (p[-1] == '%') { in trace_event_format()
3943 int ent_size = iter->ent_size; in trace_find_next_entry()
3947 * If called from ftrace_dump(), then the iter->temp buffer in trace_find_next_entry()
3952 * stamps have a large delta. See trace_print_lat_context() in trace_find_next_entry()
3954 if (iter->temp == static_temp_buf && in trace_find_next_entry()
3960 * call ring_buffer_peek() that may make the contents of iter->ent in trace_find_next_entry()
3961 * undefined. Need to copy iter->ent now. in trace_find_next_entry()
3963 if (iter->ent && iter->ent != iter->temp) { in trace_find_next_entry()
3964 if ((!iter->temp || iter->temp_size < iter->ent_size) && in trace_find_next_entry()
3965 !WARN_ON_ONCE(iter->temp == static_temp_buf)) { in trace_find_next_entry()
3967 temp = kmalloc(iter->ent_size, GFP_KERNEL); in trace_find_next_entry()
3970 kfree(iter->temp); in trace_find_next_entry()
3971 iter->temp = temp; in trace_find_next_entry()
3972 iter->temp_size = iter->ent_size; in trace_find_next_entry()
3974 memcpy(iter->temp, iter->ent, iter->ent_size); in trace_find_next_entry()
3975 iter->ent = iter->temp; in trace_find_next_entry()
3979 iter->ent_size = ent_size; in trace_find_next_entry()
3987 iter->ent = __find_next_entry(iter, &iter->cpu, in trace_find_next_entry_inc()
3988 &iter->lost_events, &iter->ts); in trace_find_next_entry_inc()
3990 if (iter->ent) in trace_find_next_entry_inc()
3993 return iter->ent ? iter : NULL; in trace_find_next_entry_inc()
3998 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts, in trace_consume()
3999 &iter->lost_events); in trace_consume()
4004 struct trace_iterator *iter = m->private; in s_next()
4008 WARN_ON_ONCE(iter->leftover); in s_next()
4013 if (iter->idx > i) in s_next()
4016 if (iter->idx < 0) in s_next()
4021 while (ent && iter->idx < i) in s_next()
4024 iter->pos = *pos; in s_next()
4035 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0; in tracing_iter_reset()
4049 if (ts >= iter->array_buffer->time_start) in tracing_iter_reset()
4057 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries; in tracing_iter_reset()
4066 struct trace_iterator *iter = m->private; in s_start()
4067 struct trace_array *tr = iter->tr; in s_start()
4068 int cpu_file = iter->cpu_file; in s_start()
4074 if (unlikely(tr->current_trace != iter->trace)) { in s_start()
4075 /* Close iter->trace before switching to the new current tracer */ in s_start()
4076 if (iter->trace->close) in s_start()
4077 iter->trace->close(iter); in s_start()
4078 iter->trace = tr->current_trace; in s_start()
4080 if (iter->trace->open) in s_start()
4081 iter->trace->open(iter); in s_start()
4086 if (iter->snapshot && iter->trace->use_max_tr) in s_start()
4087 return ERR_PTR(-EBUSY); in s_start()
4090 if (*pos != iter->pos) { in s_start()
4091 iter->ent = NULL; in s_start()
4092 iter->cpu = 0; in s_start()
4093 iter->idx = -1; in s_start()
4101 iter->leftover = 0; in s_start()
4110 if (iter->leftover) in s_start()
4113 l = *pos - 1; in s_start()
4125 struct trace_iterator *iter = m->private; in s_stop()
4128 if (iter->snapshot && iter->trace->use_max_tr) in s_stop()
4132 trace_access_unlock(iter->cpu_file); in s_stop()
4142 count = ring_buffer_entries_cpu(buf->buffer, cpu); in get_total_entries_cpu()
4148 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) { in get_total_entries_cpu()
4149 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries; in get_total_entries_cpu()
4154 ring_buffer_overrun_cpu(buf->buffer, cpu); in get_total_entries_cpu()
4182 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu); in trace_total_entries_cpu()
4194 get_total_entries(&tr->array_buffer, &total, &entries); in trace_total_entries()
4201 seq_puts(m, "# _------=> CPU# \n" in print_lat_help_header()
4202 "# / _-----=> irqs-off/BH-disabled\n" in print_lat_help_header()
4203 "# | / _----=> need-resched \n" in print_lat_help_header()
4204 "# || / _---=> hardirq/softirq \n" in print_lat_help_header()
4205 "# ||| / _--=> preempt-depth \n" in print_lat_help_header()
4206 "# |||| / _-=> migrate-disable \n" in print_lat_help_header()
4218 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n", in print_event_info()
4230 …seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : "… in print_func_help_header()
4243 seq_printf(m, "# %.*s _-----=> irqs-off/BH-disabled\n", prec, space); in print_func_help_header_irq()
4244 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space); in print_func_help_header_irq()
4245 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space); in print_func_help_header_irq()
4246 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space); in print_func_help_header_irq()
4247 seq_printf(m, "# %.*s||| / _-=> migrate-disable\n", prec, space); in print_func_help_header_irq()
4249 …seq_printf(m, "# TASK-PID %.*s CPU# ||||| TIMESTAMP FUNCTION\n", prec, " TGID … in print_func_help_header_irq()
4257 struct array_buffer *buf = iter->array_buffer; in print_trace_header()
4258 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu); in print_trace_header()
4259 struct tracer *type = iter->trace; in print_trace_header()
4262 const char *name = type->name; in print_trace_header()
4267 name, init_utsname()->release); in print_trace_header()
4268 seq_puts(m, "# -----------------------------------" in print_trace_header()
4269 "---------------------------------\n"); in print_trace_header()
4272 nsecs_to_usecs(data->saved_latency), in print_trace_header()
4275 buf->cpu, in print_trace_header()
4288 seq_puts(m, "# -----------------\n"); in print_trace_header()
4289 seq_printf(m, "# | task: %.16s-%d " in print_trace_header()
4291 data->comm, data->pid, in print_trace_header()
4292 from_kuid_munged(seq_user_ns(m), data->uid), data->nice, in print_trace_header()
4293 data->policy, data->rt_priority); in print_trace_header()
4294 seq_puts(m, "# -----------------\n"); in print_trace_header()
4296 if (data->critical_start) { in print_trace_header()
4298 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); in print_trace_header()
4299 trace_print_seq(m, &iter->seq); in print_trace_header()
4301 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); in print_trace_header()
4302 trace_print_seq(m, &iter->seq); in print_trace_header()
4311 struct trace_seq *s = &iter->seq; in test_cpu_buff_start()
4312 struct trace_array *tr = iter->tr; in test_cpu_buff_start()
4314 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE)) in test_cpu_buff_start()
4317 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE)) in test_cpu_buff_start()
4320 if (cpumask_available(iter->started) && in test_cpu_buff_start()
4321 cpumask_test_cpu(iter->cpu, iter->started)) in test_cpu_buff_start()
4324 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries) in test_cpu_buff_start()
4327 if (cpumask_available(iter->started)) in test_cpu_buff_start()
4328 cpumask_set_cpu(iter->cpu, iter->started); in test_cpu_buff_start()
4331 if (iter->idx > 1) in test_cpu_buff_start()
4333 iter->cpu); in test_cpu_buff_start()
4338 struct trace_array *tr = iter->tr; in print_trace_fmt()
4339 struct trace_seq *s = &iter->seq; in print_trace_fmt()
4340 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK); in print_trace_fmt()
4344 entry = iter->ent; in print_trace_fmt()
4348 event = ftrace_find_event(entry->type); in print_trace_fmt()
4350 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { in print_trace_fmt()
4351 if (iter->iter_flags & TRACE_FILE_LAT_FMT) in print_trace_fmt()
4361 if (tr->trace_flags & TRACE_ITER_FIELDS) in print_trace_fmt()
4363 return event->funcs->trace(iter, sym_flags, event); in print_trace_fmt()
4366 trace_seq_printf(s, "Unknown type %d\n", entry->type); in print_trace_fmt()
4373 struct trace_array *tr = iter->tr; in print_raw_fmt()
4374 struct trace_seq *s = &iter->seq; in print_raw_fmt()
4378 entry = iter->ent; in print_raw_fmt()
4380 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) in print_raw_fmt()
4382 entry->pid, iter->cpu, iter->ts); in print_raw_fmt()
4387 event = ftrace_find_event(entry->type); in print_raw_fmt()
4389 return event->funcs->raw(iter, 0, event); in print_raw_fmt()
4391 trace_seq_printf(s, "%d ?\n", entry->type); in print_raw_fmt()
4398 struct trace_array *tr = iter->tr; in print_hex_fmt()
4399 struct trace_seq *s = &iter->seq; in print_hex_fmt()
4404 entry = iter->ent; in print_hex_fmt()
4406 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { in print_hex_fmt()
4407 SEQ_PUT_HEX_FIELD(s, entry->pid); in print_hex_fmt()
4408 SEQ_PUT_HEX_FIELD(s, iter->cpu); in print_hex_fmt()
4409 SEQ_PUT_HEX_FIELD(s, iter->ts); in print_hex_fmt()
4414 event = ftrace_find_event(entry->type); in print_hex_fmt()
4416 enum print_line_t ret = event->funcs->hex(iter, 0, event); in print_hex_fmt()
4428 struct trace_array *tr = iter->tr; in print_bin_fmt()
4429 struct trace_seq *s = &iter->seq; in print_bin_fmt()
4433 entry = iter->ent; in print_bin_fmt()
4435 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { in print_bin_fmt()
4436 SEQ_PUT_FIELD(s, entry->pid); in print_bin_fmt()
4437 SEQ_PUT_FIELD(s, iter->cpu); in print_bin_fmt()
4438 SEQ_PUT_FIELD(s, iter->ts); in print_bin_fmt()
4443 event = ftrace_find_event(entry->type); in print_bin_fmt()
4444 return event ? event->funcs->binary(iter, 0, event) : in print_bin_fmt()
4454 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { in trace_empty()
4455 cpu = iter->cpu_file; in trace_empty()
4461 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu)) in trace_empty()
4473 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu)) in trace_empty()
4484 struct trace_array *tr = iter->tr; in print_trace_line()
4485 unsigned long trace_flags = tr->trace_flags; in print_trace_line()
4488 if (iter->lost_events) { in print_trace_line()
4489 if (iter->lost_events == (unsigned long)-1) in print_trace_line()
4490 trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n", in print_trace_line()
4491 iter->cpu); in print_trace_line()
4493 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", in print_trace_line()
4494 iter->cpu, iter->lost_events); in print_trace_line()
4495 if (trace_seq_has_overflowed(&iter->seq)) in print_trace_line()
4499 if (iter->trace && iter->trace->print_line) { in print_trace_line()
4500 ret = iter->trace->print_line(iter); in print_trace_line()
4505 if (iter->ent->type == TRACE_BPUTS && in print_trace_line()
4510 if (iter->ent->type == TRACE_BPRINT && in print_trace_line()
4515 if (iter->ent->type == TRACE_PRINT && in print_trace_line()
4534 struct trace_iterator *iter = m->private; in trace_latency_header()
4535 struct trace_array *tr = iter->tr; in trace_latency_header()
4541 if (iter->iter_flags & TRACE_FILE_LAT_FMT) in trace_latency_header()
4544 if (!(tr->trace_flags & TRACE_ITER_VERBOSE)) in trace_latency_header()
4550 struct trace_iterator *iter = m->private; in trace_default_header()
4551 struct trace_array *tr = iter->tr; in trace_default_header()
4552 unsigned long trace_flags = tr->trace_flags; in trace_default_header()
4557 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { in trace_default_header()
4567 print_func_help_header_irq(iter->array_buffer, in trace_default_header()
4570 print_func_help_header(iter->array_buffer, m, in trace_default_header()
4612 if (iter->tr->allocated_snapshot) in print_snapshot_help()
4618 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) in print_snapshot_help()
4633 if (iter->ent == NULL) { in s_show()
4634 if (iter->tr) { in s_show()
4635 seq_printf(m, "# tracer: %s\n", iter->trace->name); in s_show()
4639 if (iter->snapshot && trace_empty(iter)) in s_show()
4641 else if (iter->trace && iter->trace->print_header) in s_show()
4642 iter->trace->print_header(m); in s_show()
4646 } else if (iter->leftover) { in s_show()
4651 ret = trace_print_seq(m, &iter->seq); in s_show()
4654 iter->leftover = ret; in s_show()
4659 iter->seq.full = 0; in s_show()
4660 trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n"); in s_show()
4662 ret = trace_print_seq(m, &iter->seq); in s_show()
4668 * -1 otherwise. in s_show()
4670 iter->leftover = ret; in s_show()
4682 if (inode->i_cdev) /* See trace_create_cpu_file() */ in tracing_get_cpu()
4683 return (long)inode->i_cdev - 1; in tracing_get_cpu()
4704 if (iter->fmt != static_fmt_buf) in free_trace_iter_content()
4705 kfree(iter->fmt); in free_trace_iter_content()
4707 kfree(iter->temp); in free_trace_iter_content()
4708 kfree(iter->buffer_iter); in free_trace_iter_content()
4709 mutex_destroy(&iter->mutex); in free_trace_iter_content()
4710 free_cpumask_var(iter->started); in free_trace_iter_content()
4716 struct trace_array *tr = inode->i_private; in __tracing_open()
4721 return ERR_PTR(-ENODEV); in __tracing_open()
4725 return ERR_PTR(-ENOMEM); in __tracing_open()
4727 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter), in __tracing_open()
4729 if (!iter->buffer_iter) in __tracing_open()
4733 * trace_find_next_entry() may need to save off iter->ent. in __tracing_open()
4734 * It will place it into the iter->temp buffer. As most in __tracing_open()
4737 * allocate a new buffer to adjust for the bigger iter->ent. in __tracing_open()
4740 iter->temp = kmalloc(128, GFP_KERNEL); in __tracing_open()
4741 if (iter->temp) in __tracing_open()
4742 iter->temp_size = 128; in __tracing_open()
4751 iter->fmt = NULL; in __tracing_open()
4752 iter->fmt_size = 0; in __tracing_open()
4755 iter->trace = tr->current_trace; in __tracing_open()
4757 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL)) in __tracing_open()
4760 iter->tr = tr; in __tracing_open()
4764 if (tr->current_trace->print_max || snapshot) in __tracing_open()
4765 iter->array_buffer = &tr->max_buffer; in __tracing_open()
4768 iter->array_buffer = &tr->array_buffer; in __tracing_open()
4769 iter->snapshot = snapshot; in __tracing_open()
4770 iter->pos = -1; in __tracing_open()
4771 iter->cpu_file = tracing_get_cpu(inode); in __tracing_open()
4772 mutex_init(&iter->mutex); in __tracing_open()
4775 if (iter->trace->open) in __tracing_open()
4776 iter->trace->open(iter); in __tracing_open()
4779 if (ring_buffer_overruns(iter->array_buffer->buffer)) in __tracing_open()
4780 iter->iter_flags |= TRACE_FILE_ANNOTATE; in __tracing_open()
4783 if (trace_clocks[tr->clock_id].in_ns) in __tracing_open()
4784 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; in __tracing_open()
4787 * If pause-on-trace is enabled, then stop the trace while in __tracing_open()
4790 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE)) in __tracing_open()
4793 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { in __tracing_open()
4795 iter->buffer_iter[cpu] = in __tracing_open()
4796 ring_buffer_read_prepare(iter->array_buffer->buffer, in __tracing_open()
4801 ring_buffer_read_start(iter->buffer_iter[cpu]); in __tracing_open()
4805 cpu = iter->cpu_file; in __tracing_open()
4806 iter->buffer_iter[cpu] = in __tracing_open()
4807 ring_buffer_read_prepare(iter->array_buffer->buffer, in __tracing_open()
4810 ring_buffer_read_start(iter->buffer_iter[cpu]); in __tracing_open()
4823 return ERR_PTR(-ENOMEM); in __tracing_open()
4834 filp->private_data = inode->i_private; in tracing_open_generic()
4849 struct trace_array *tr = inode->i_private; in tracing_open_generic_tr()
4856 filp->private_data = inode->i_private; in tracing_open_generic_tr()
4867 struct trace_event_file *file = inode->i_private; in tracing_open_file_tr()
4870 ret = tracing_check_open_get_tr(file->tr); in tracing_open_file_tr()
4877 if (file->flags & EVENT_FILE_FL_FREED) { in tracing_open_file_tr()
4878 trace_array_put(file->tr); in tracing_open_file_tr()
4879 ret = -ENODEV; in tracing_open_file_tr()
4888 filp->private_data = inode->i_private; in tracing_open_file_tr()
4895 struct trace_event_file *file = inode->i_private; in tracing_release_file_tr()
4897 trace_array_put(file->tr); in tracing_release_file_tr()
4917 struct trace_array *tr = inode->i_private; in tracing_release()
4918 struct seq_file *m = file->private_data; in tracing_release()
4922 if (!(file->f_mode & FMODE_READ)) { in tracing_release()
4928 iter = m->private; in tracing_release()
4932 if (iter->buffer_iter[cpu]) in tracing_release()
4933 ring_buffer_read_finish(iter->buffer_iter[cpu]); in tracing_release()
4936 if (iter->trace && iter->trace->close) in tracing_release()
4937 iter->trace->close(iter); in tracing_release()
4939 if (!iter->snapshot && tr->stop_count) in tracing_release()
4955 struct trace_array *tr = inode->i_private; in tracing_release_generic_tr()
4963 struct trace_array *tr = inode->i_private; in tracing_single_release_tr()
4972 struct trace_array *tr = inode->i_private; in tracing_open()
4981 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { in tracing_open()
4983 struct array_buffer *trace_buf = &tr->array_buffer; in tracing_open()
4986 if (tr->current_trace->print_max) in tracing_open()
4987 trace_buf = &tr->max_buffer; in tracing_open()
4996 if (file->f_mode & FMODE_READ) { in tracing_open()
5000 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) in tracing_open()
5001 iter->iter_flags |= TRACE_FILE_LAT_FMT; in tracing_open()
5020 if (tr->range_addr_start && t->use_max_tr) in trace_ok_for_array()
5023 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances; in trace_ok_for_array()
5031 t = t->next; in get_tracer_for_array()
5039 struct trace_array *tr = m->private; in t_next()
5045 t = get_tracer_for_array(tr, t->next); in t_next()
5052 struct trace_array *tr = m->private; in t_start()
5077 seq_puts(m, t->name); in t_show()
5078 if (t->next) in t_show()
5095 struct trace_array *tr = inode->i_private; in show_traces_open()
5109 m = file->private_data; in show_traces_open()
5110 m->private = tr; in show_traces_open()
5117 struct trace_array *tr = inode->i_private; in tracing_seq_release()
5134 if (file->f_mode & FMODE_READ) in tracing_lseek()
5137 file->f_pos = ret = 0; in tracing_lseek()
5163 struct trace_array *tr = file_inode(filp)->i_private; in tracing_cpumask_read()
5168 cpumask_pr_args(tr->tracing_cpumask)) + 1; in tracing_cpumask_read()
5171 return -ENOMEM; in tracing_cpumask_read()
5174 cpumask_pr_args(tr->tracing_cpumask)); in tracing_cpumask_read()
5176 count = -EINVAL; in tracing_cpumask_read()
5193 return -EINVAL; in tracing_set_cpumask()
5196 arch_spin_lock(&tr->max_lock); in tracing_set_cpumask()
5202 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) && in tracing_set_cpumask()
5204 atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled); in tracing_set_cpumask()
5205 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu); in tracing_set_cpumask()
5207 ring_buffer_record_disable_cpu(tr->max_buffer.buffer, cpu); in tracing_set_cpumask()
5210 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) && in tracing_set_cpumask()
5212 atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled); in tracing_set_cpumask()
5213 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu); in tracing_set_cpumask()
5215 ring_buffer_record_enable_cpu(tr->max_buffer.buffer, cpu); in tracing_set_cpumask()
5219 arch_spin_unlock(&tr->max_lock); in tracing_set_cpumask()
5222 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new); in tracing_set_cpumask()
5231 struct trace_array *tr = file_inode(filp)->i_private; in tracing_cpumask_write()
5236 return -ENOMEM; in tracing_cpumask_write()
5267 struct trace_array *tr = m->private; in tracing_trace_options_show()
5272 tracer_flags = tr->current_trace->flags->val; in tracing_trace_options_show()
5273 trace_opts = tr->current_trace->flags->opts; in tracing_trace_options_show()
5276 if (tr->trace_flags & (1 << i)) in tracing_trace_options_show()
5297 struct tracer *trace = tracer_flags->trace; in __set_tracer_option()
5300 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg); in __set_tracer_option()
5305 tracer_flags->val &= ~opts->bit; in __set_tracer_option()
5307 tracer_flags->val |= opts->bit; in __set_tracer_option()
5314 struct tracer *trace = tr->current_trace; in set_tracer_option()
5315 struct tracer_flags *tracer_flags = trace->flags; in set_tracer_option()
5319 for (i = 0; tracer_flags->opts[i].name; i++) { in set_tracer_option()
5320 opts = &tracer_flags->opts[i]; in set_tracer_option()
5322 if (strcmp(cmp, opts->name) == 0) in set_tracer_option()
5323 return __set_tracer_option(tr, trace->flags, opts, neg); in set_tracer_option()
5326 return -EINVAL; in set_tracer_option()
5332 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set) in trace_keep_overwrite()
5333 return -1; in trace_keep_overwrite()
5346 if (!!(tr->trace_flags & mask) == !!enabled) in set_tracer_flag()
5350 if (tr->current_trace->flag_changed) in set_tracer_flag()
5351 if (tr->current_trace->flag_changed(tr, mask, !!enabled)) in set_tracer_flag()
5352 return -EINVAL; in set_tracer_flag()
5363 return -EINVAL; in set_tracer_flag()
5374 tr->trace_flags |= mask; in set_tracer_flag()
5376 tr->trace_flags &= ~mask; in set_tracer_flag()
5384 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID; in set_tracer_flag()
5385 return -ENOMEM; in set_tracer_flag()
5398 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled); in set_tracer_flag()
5400 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled); in set_tracer_flag()
5431 ret = match_string(trace_options, -1, cmp); in trace_set_options()
5467 *(buf - 1) = ','; in apply_trace_boot_options()
5475 struct seq_file *m = filp->private_data; in tracing_trace_options_write()
5476 struct trace_array *tr = m->private; in tracing_trace_options_write()
5481 return -EINVAL; in tracing_trace_options_write()
5484 return -EFAULT; in tracing_trace_options_write()
5499 struct trace_array *tr = inode->i_private; in tracing_trace_options_open()
5506 ret = single_open(file, tracing_trace_options_show, inode->i_private); in tracing_trace_options_open()
5522 "tracing mini-HOWTO:\n\n"
5526 "\t e.g. mount -t tracefs [-o [gid=<gid>]] nodev /sys/kernel/tracing\n\n"
5528 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
5530 " trace\t\t\t- The static contents of the buffer\n"
5532 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
5533 " current_tracer\t- function and latency tracers\n"
5534 " available_tracers\t- list of configured tracers for current_tracer\n"
5535 " error_log\t- error log for failed commands (that support it)\n"
5536 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
5537 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
5538 " trace_clock\t\t- change the clock used to order events\n"
5545 " x86-tsc: TSC cycle counter\n"
5547 "\n timestamp_mode\t- view the mode used to timestamp events\n"
5548 " delta: Delta difference against a buffer-wide timestamp\n"
5550 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
5551 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
5552 " tracing_cpumask\t- Limit which CPUs to trace\n"
5553 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
5554 "\t\t\t Remove sub-buffer with rmdir\n"
5555 " trace_options\t\t- Set format or modify how tracing happens\n"
5558 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
5560 "\n available_filter_functions - list of functions that can be filtered on\n"
5561 " set_ftrace_filter\t- echo function name in here to only trace these\n"
5563 "\t accepts: func_full_name or glob-matching-pattern\n"
5565 "\t Format: :mod:<module-name>\n"
5592 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
5598 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
5600 " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
5604 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
5605 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
5606 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
5609 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
5614 " stack_trace\t\t- Shows the max stack trace when active\n"
5615 " stack_max_size\t- Shows current max stack size that was traced\n"
5619 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
5624 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
5628 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
5632 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
5637 "\t accepts: event-definitions (one definition per line)\n"
5643 "\t f[:[<group>/][<event>]] <func-name>[%return] [<args>]\n"
5649 "\t e[:[<group>/][<event>]] <attached-group>.<attached-event> [<args>] [if <filter>]\n"
5650 "\t -:[<group>/][<event>]\n"
5659 "\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n"
5663 "\t <argname>[->field[->field|.field...]],\n"
5668 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
5671 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
5672 "\t symstr, %pd/%pD, <type>\\[<array-size>\\]\n"
5679 "\t of the <attached-group>/<attached-event>.\n"
5681 " events/\t\t- Directory containing all trace event subsystems:\n"
5682 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
5683 " events/<system>/\t- Directory containing all trace events for <system>:\n"
5684 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
5686 " filter\t\t- If set, only events passing filter are traced\n"
5687 " events/<system>/<event>/\t- Directory containing control files for\n"
5689 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
5690 " filter\t\t- If set, only events passing filter are traced\n"
5691 " trigger\t\t- If set, a command to perform when event is hit\n"
5725 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
5737 "\t common_timestamp - to record current timestamp\n"
5738 "\t common_cpu - to record the CPU the event happened on\n"
5741 "\t - a reference to a field e.g. x=current_timestamp,\n"
5742 "\t - a reference to another variable e.g. y=$x,\n"
5743 "\t - a numeric literal: e.g. ms_per_sec=1000,\n"
5744 "\t - an arithmetic expression: e.g. time_secs=current_timestamp/1000\n"
5746 "\t hist trigger arithmetic expressions support addition(+), subtraction(-),\n"
5775 "\t .sym-offset display an address as a symbol and offset\n"
5782 "\t .graph display a bar-graph of a value\n\n"
5794 "\t already-attached hist trigger. The syntax is analogous to\n"
5800 "\t onmatch(matching.event) - invoke on addition or update\n"
5801 "\t onmax(var) - invoke if var exceeds current max\n"
5802 "\t onchange(var) - invoke action if var changes\n\n"
5804 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
5805 "\t save(field,...) - save current event fields\n"
5807 "\t snapshot() - snapshot the trace buffer\n\n"
5810 " events/synthetic_events\t- Create/append/remove/show synthetic events\n"
5835 if (!ptr->map.eval_string) { in update_eval_map()
5836 if (ptr->tail.next) { in update_eval_map()
5837 ptr = ptr->tail.next; in update_eval_map()
5893 ptr->map.eval_string, ptr->map.eval_value, in eval_map_show()
5894 ptr->map.system); in eval_map_show()
5928 return ptr + ptr->head.length + 1; in trace_eval_jmp_to_tail()
5961 if (!ptr->tail.next) in trace_insert_eval_map_file()
5963 ptr = ptr->tail.next; in trace_insert_eval_map_file()
5966 ptr->tail.next = map_array; in trace_insert_eval_map_file()
5968 map_array->head.mod = mod; in trace_insert_eval_map_file()
5969 map_array->head.length = len; in trace_insert_eval_map_file()
5973 map_array->map = **map; in trace_insert_eval_map_file()
6012 struct trace_array *tr = filp->private_data; in tracing_set_trace_read()
6017 r = sprintf(buf, "%s\n", tr->current_trace->name); in tracing_set_trace_read()
6025 tracing_reset_online_cpus(&tr->array_buffer); in tracer_init()
6026 return t->init(tr); in tracer_init()
6034 per_cpu_ptr(buf->data, cpu)->entries = val; in set_buffer_entries()
6040 set_buffer_entries(buf, ring_buffer_size(buf->buffer, 0)); in update_buffer_entries()
6042 per_cpu_ptr(buf->data, cpu)->entries = ring_buffer_size(buf->buffer, cpu); in update_buffer_entries()
6055 ret = ring_buffer_resize(trace_buf->buffer, in resize_buffer_duplicate_size()
6056 per_cpu_ptr(size_buf->data, cpu)->entries, cpu); in resize_buffer_duplicate_size()
6059 per_cpu_ptr(trace_buf->data, cpu)->entries = in resize_buffer_duplicate_size()
6060 per_cpu_ptr(size_buf->data, cpu)->entries; in resize_buffer_duplicate_size()
6063 ret = ring_buffer_resize(trace_buf->buffer, in resize_buffer_duplicate_size()
6064 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id); in resize_buffer_duplicate_size()
6066 per_cpu_ptr(trace_buf->data, cpu_id)->entries = in resize_buffer_duplicate_size()
6067 per_cpu_ptr(size_buf->data, cpu_id)->entries; in resize_buffer_duplicate_size()
6087 if (!tr->array_buffer.buffer) in __tracing_resize_ring_buffer()
6093 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu); in __tracing_resize_ring_buffer()
6098 if (!tr->allocated_snapshot) in __tracing_resize_ring_buffer()
6101 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu); in __tracing_resize_ring_buffer()
6103 int r = resize_buffer_duplicate_size(&tr->array_buffer, in __tracing_resize_ring_buffer()
6104 &tr->array_buffer, cpu); in __tracing_resize_ring_buffer()
6126 update_buffer_entries(&tr->max_buffer, cpu); in __tracing_resize_ring_buffer()
6131 update_buffer_entries(&tr->array_buffer, cpu); in __tracing_resize_ring_buffer()
6147 ret = -EINVAL; in tracing_resize_ring_buffer()
6154 ret = -ENOMEM; in tracing_resize_ring_buffer()
6164 if (!tr->text_delta && !tr->data_delta) in update_last_data()
6173 tracing_reset_all_cpus(&tr->array_buffer); in update_last_data()
6176 tr->text_delta = 0; in update_last_data()
6177 tr->data_delta = 0; in update_last_data()
6181 * tracing_update_buffers - used by tracing facility to expand ring buffers
6199 if (!tr->ring_buffer_expanded) in tracing_update_buffers()
6218 if (tr->current_trace == &nop_trace) in tracing_set_nop()
6221 tr->current_trace->enabled--; in tracing_set_nop()
6223 if (tr->current_trace->reset) in tracing_set_nop()
6224 tr->current_trace->reset(tr); in tracing_set_nop()
6226 tr->current_trace = &nop_trace; in tracing_set_nop()
6234 if (!tr->dir) in add_tracer_options()
6256 if (!tr->ring_buffer_expanded) { in tracing_set_tracer()
6264 for (t = trace_types; t; t = t->next) { in tracing_set_tracer()
6265 if (strcmp(t->name, buf) == 0) in tracing_set_tracer()
6269 ret = -EINVAL; in tracing_set_tracer()
6272 if (t == tr->current_trace) in tracing_set_tracer()
6276 if (t->use_max_tr) { in tracing_set_tracer()
6278 arch_spin_lock(&tr->max_lock); in tracing_set_tracer()
6279 if (tr->cond_snapshot) in tracing_set_tracer()
6280 ret = -EBUSY; in tracing_set_tracer()
6281 arch_spin_unlock(&tr->max_lock); in tracing_set_tracer()
6288 if (system_state < SYSTEM_RUNNING && t->noboot) { in tracing_set_tracer()
6290 t->name); in tracing_set_tracer()
6296 ret = -EINVAL; in tracing_set_tracer()
6301 if (tr->trace_ref) { in tracing_set_tracer()
6302 ret = -EBUSY; in tracing_set_tracer()
6308 tr->current_trace->enabled--; in tracing_set_tracer()
6310 if (tr->current_trace->reset) in tracing_set_tracer()
6311 tr->current_trace->reset(tr); in tracing_set_tracer()
6314 had_max_tr = tr->current_trace->use_max_tr; in tracing_set_tracer()
6317 tr->current_trace = &nop_trace; in tracing_set_tracer()
6319 if (had_max_tr && !t->use_max_tr) { in tracing_set_tracer()
6332 if (!had_max_tr && t->use_max_tr) { in tracing_set_tracer()
6338 tr->current_trace = &nop_trace; in tracing_set_tracer()
6341 if (t->init) { in tracing_set_tracer()
6345 if (t->use_max_tr) in tracing_set_tracer()
6352 tr->current_trace = t; in tracing_set_tracer()
6353 tr->current_trace->enabled++; in tracing_set_tracer()
6365 struct trace_array *tr = filp->private_data; in tracing_set_trace_write()
6377 return -EFAULT; in tracing_set_trace_write()
6400 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr)); in tracing_nsecs_read()
6433 struct trace_array *tr = filp->private_data; in tracing_thresh_write()
6441 if (tr->current_trace->update_thresh) { in tracing_thresh_write()
6442 ret = tr->current_trace->update_thresh(tr); in tracing_thresh_write()
6460 struct trace_array *tr = filp->private_data; in tracing_max_lat_read()
6462 return tracing_nsecs_read(&tr->max_latency, ubuf, cnt, ppos); in tracing_max_lat_read()
6469 struct trace_array *tr = filp->private_data; in tracing_max_lat_write()
6471 return tracing_nsecs_write(&tr->max_latency, ubuf, cnt, ppos); in tracing_max_lat_write()
6479 if (cpumask_empty(tr->pipe_cpumask)) { in open_pipe_on_cpu()
6480 cpumask_setall(tr->pipe_cpumask); in open_pipe_on_cpu()
6483 } else if (!cpumask_test_cpu(cpu, tr->pipe_cpumask)) { in open_pipe_on_cpu()
6484 cpumask_set_cpu(cpu, tr->pipe_cpumask); in open_pipe_on_cpu()
6487 return -EBUSY; in open_pipe_on_cpu()
6493 WARN_ON(!cpumask_full(tr->pipe_cpumask)); in close_pipe_on_cpu()
6494 cpumask_clear(tr->pipe_cpumask); in close_pipe_on_cpu()
6496 WARN_ON(!cpumask_test_cpu(cpu, tr->pipe_cpumask)); in close_pipe_on_cpu()
6497 cpumask_clear_cpu(cpu, tr->pipe_cpumask); in close_pipe_on_cpu()
6503 struct trace_array *tr = inode->i_private; in tracing_open_pipe()
6521 ret = -ENOMEM; in tracing_open_pipe()
6525 trace_seq_init(&iter->seq); in tracing_open_pipe()
6526 iter->trace = tr->current_trace; in tracing_open_pipe()
6528 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { in tracing_open_pipe()
6529 ret = -ENOMEM; in tracing_open_pipe()
6534 cpumask_setall(iter->started); in tracing_open_pipe()
6536 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) in tracing_open_pipe()
6537 iter->iter_flags |= TRACE_FILE_LAT_FMT; in tracing_open_pipe()
6540 if (trace_clocks[tr->clock_id].in_ns) in tracing_open_pipe()
6541 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; in tracing_open_pipe()
6543 iter->tr = tr; in tracing_open_pipe()
6544 iter->array_buffer = &tr->array_buffer; in tracing_open_pipe()
6545 iter->cpu_file = cpu; in tracing_open_pipe()
6546 mutex_init(&iter->mutex); in tracing_open_pipe()
6547 filp->private_data = iter; in tracing_open_pipe()
6549 if (iter->trace->pipe_open) in tracing_open_pipe()
6550 iter->trace->pipe_open(iter); in tracing_open_pipe()
6554 tr->trace_ref++; in tracing_open_pipe()
6571 struct trace_iterator *iter = file->private_data; in tracing_release_pipe()
6572 struct trace_array *tr = inode->i_private; in tracing_release_pipe()
6576 tr->trace_ref--; in tracing_release_pipe()
6578 if (iter->trace->pipe_close) in tracing_release_pipe()
6579 iter->trace->pipe_close(iter); in tracing_release_pipe()
6580 close_pipe_on_cpu(tr, iter->cpu_file); in tracing_release_pipe()
6594 struct trace_array *tr = iter->tr; in trace_poll()
6597 if (trace_buffer_iter(iter, iter->cpu_file)) in trace_poll()
6600 if (tr->trace_flags & TRACE_ITER_BLOCK) in trace_poll()
6606 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file, in trace_poll()
6607 filp, poll_table, iter->tr->buffer_percent); in trace_poll()
6613 struct trace_iterator *iter = filp->private_data; in tracing_poll_pipe()
6618 /* Must be called with iter->mutex held. */
6621 struct trace_iterator *iter = filp->private_data; in tracing_wait_pipe()
6626 if ((filp->f_flags & O_NONBLOCK)) { in tracing_wait_pipe()
6627 return -EAGAIN; in tracing_wait_pipe()
6637 * iter->pos will be 0 if we haven't read anything. in tracing_wait_pipe()
6639 if (!tracer_tracing_is_on(iter->tr) && iter->pos) in tracing_wait_pipe()
6642 mutex_unlock(&iter->mutex); in tracing_wait_pipe()
6646 mutex_lock(&iter->mutex); in tracing_wait_pipe()
6662 struct trace_iterator *iter = filp->private_data; in tracing_read_pipe()
6670 mutex_lock(&iter->mutex); in tracing_read_pipe()
6673 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); in tracing_read_pipe()
6674 if (sret != -EBUSY) in tracing_read_pipe()
6677 trace_seq_init(&iter->seq); in tracing_read_pipe()
6679 if (iter->trace->read) { in tracing_read_pipe()
6680 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); in tracing_read_pipe()
6697 cnt = TRACE_SEQ_BUFFER_SIZE - 1; in tracing_read_pipe()
6701 cpumask_clear(iter->started); in tracing_read_pipe()
6702 trace_seq_init(&iter->seq); in tracing_read_pipe()
6705 trace_access_lock(iter->cpu_file); in tracing_read_pipe()
6708 int save_len = iter->seq.seq.len; in tracing_read_pipe()
6714 * trace_seq_to_user() will returns -EBUSY because save_len == 0, in tracing_read_pipe()
6719 iter->seq.full = 0; in tracing_read_pipe()
6720 trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n"); in tracing_read_pipe()
6726 iter->seq.seq.len = save_len; in tracing_read_pipe()
6732 if (trace_seq_used(&iter->seq) >= cnt) in tracing_read_pipe()
6740 WARN_ONCE(iter->seq.full, "full flag set for trace type %d", in tracing_read_pipe()
6741 iter->ent->type); in tracing_read_pipe()
6743 trace_access_unlock(iter->cpu_file); in tracing_read_pipe()
6747 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); in tracing_read_pipe()
6748 if (iter->seq.readpos >= trace_seq_used(&iter->seq)) in tracing_read_pipe()
6749 trace_seq_init(&iter->seq); in tracing_read_pipe()
6755 if (sret == -EBUSY) in tracing_read_pipe()
6759 mutex_unlock(&iter->mutex); in tracing_read_pipe()
6767 __free_page(spd->pages[idx]); in tracing_spd_release_pipe()
6777 /* Seq buffer is page-sized, exactly what we need. */ in tracing_fill_pipe_page()
6779 save_len = iter->seq.seq.len; in tracing_fill_pipe_page()
6782 if (trace_seq_has_overflowed(&iter->seq)) { in tracing_fill_pipe_page()
6783 iter->seq.seq.len = save_len; in tracing_fill_pipe_page()
6789 * be set if the iter->seq overflowed. But check it in tracing_fill_pipe_page()
6793 iter->seq.seq.len = save_len; in tracing_fill_pipe_page()
6797 count = trace_seq_used(&iter->seq) - save_len; in tracing_fill_pipe_page()
6800 iter->seq.seq.len = save_len; in tracing_fill_pipe_page()
6806 rem -= count; in tracing_fill_pipe_page()
6809 iter->ent = NULL; in tracing_fill_pipe_page()
6825 struct trace_iterator *iter = filp->private_data; in tracing_splice_read_pipe()
6839 return -ENOMEM; in tracing_splice_read_pipe()
6841 mutex_lock(&iter->mutex); in tracing_splice_read_pipe()
6843 if (iter->trace->splice_read) { in tracing_splice_read_pipe()
6844 ret = iter->trace->splice_read(iter, filp, in tracing_splice_read_pipe()
6854 if (!iter->ent && !trace_find_next_entry_inc(iter)) { in tracing_splice_read_pipe()
6855 ret = -EFAULT; in tracing_splice_read_pipe()
6860 trace_access_lock(iter->cpu_file); in tracing_splice_read_pipe()
6871 ret = trace_seq_to_buffer(&iter->seq, in tracing_splice_read_pipe()
6873 trace_seq_used(&iter->seq)); in tracing_splice_read_pipe()
6879 spd.partial[i].len = trace_seq_used(&iter->seq); in tracing_splice_read_pipe()
6881 trace_seq_init(&iter->seq); in tracing_splice_read_pipe()
6884 trace_access_unlock(iter->cpu_file); in tracing_splice_read_pipe()
6886 mutex_unlock(&iter->mutex); in tracing_splice_read_pipe()
6899 mutex_unlock(&iter->mutex); in tracing_splice_read_pipe()
6908 struct trace_array *tr = inode->i_private; in tracing_entries_read()
6926 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries; in tracing_entries_read()
6927 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) { in tracing_entries_read()
6934 if (!tr->ring_buffer_expanded) in tracing_entries_read()
6943 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10); in tracing_entries_read()
6956 struct trace_array *tr = inode->i_private; in tracing_entries_write()
6966 return -EINVAL; in tracing_entries_write()
6983 struct trace_array *tr = filp->private_data; in tracing_total_entries_read()
6990 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10; in tracing_total_entries_read()
6991 if (!tr->ring_buffer_expanded) in tracing_total_entries_read()
6994 if (tr->ring_buffer_expanded) in tracing_total_entries_read()
7006 struct trace_array *tr = filp->private_data; in tracing_last_boot_read()
7012 seq_buf_printf(&seq, "text delta:\t%ld\n", tr->text_delta); in tracing_last_boot_read()
7013 seq_buf_printf(&seq, "data delta:\t%ld\n", tr->data_delta); in tracing_last_boot_read()
7020 struct trace_array *tr = inode->i_private; in tracing_buffer_meta_open()
7028 ret = ring_buffer_meta_seq_init(filp, tr->array_buffer.buffer, cpu); in tracing_buffer_meta_open()
7051 struct trace_array *tr = inode->i_private; in tracing_free_buffer_release()
7054 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE) in tracing_free_buffer_release()
7070 struct trace_array *tr = filp->private_data; in tracing_mark_write()
7082 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */ in tracing_mark_write()
7085 return -EINVAL; in tracing_mark_write()
7087 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) in tracing_mark_write()
7088 return -EINVAL; in tracing_mark_write()
7091 return -EINVAL; in tracing_mark_write()
7102 size += FAULTED_SIZE - cnt; in tracing_mark_write()
7104 buffer = tr->array_buffer.buffer; in tracing_mark_write()
7115 return -EBADF; in tracing_mark_write()
7116 cnt = ring_buffer_max_event_size(buffer) - meta_size; in tracing_mark_write()
7119 return -EBADF; in tracing_mark_write()
7124 return -EBADF; in tracing_mark_write()
7128 entry->ip = _THIS_IP_; in tracing_mark_write()
7130 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt); in tracing_mark_write()
7132 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE); in tracing_mark_write()
7134 written = -EFAULT; in tracing_mark_write()
7138 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) { in tracing_mark_write()
7140 entry->buf[cnt] = '\0'; in tracing_mark_write()
7141 tt = event_triggers_call(tr->trace_marker_file, buffer, entry, event); in tracing_mark_write()
7144 if (entry->buf[cnt - 1] != '\n') { in tracing_mark_write()
7145 entry->buf[cnt] = '\n'; in tracing_mark_write()
7146 entry->buf[cnt + 1] = '\0'; in tracing_mark_write()
7148 entry->buf[cnt] = '\0'; in tracing_mark_write()
7155 event_triggers_post_call(tr->trace_marker_file, tt); in tracing_mark_write()
7164 struct trace_array *tr = filp->private_data; in tracing_mark_raw_write()
7175 return -EINVAL; in tracing_mark_raw_write()
7177 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) in tracing_mark_raw_write()
7178 return -EINVAL; in tracing_mark_raw_write()
7182 return -EINVAL; in tracing_mark_raw_write()
7186 size += FAULT_SIZE_ID - cnt; in tracing_mark_raw_write()
7188 buffer = tr->array_buffer.buffer; in tracing_mark_raw_write()
7191 return -EINVAL; in tracing_mark_raw_write()
7197 return -EBADF; in tracing_mark_raw_write()
7201 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt); in tracing_mark_raw_write()
7203 entry->id = -1; in tracing_mark_raw_write()
7204 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE); in tracing_mark_raw_write()
7205 written = -EFAULT; in tracing_mark_raw_write()
7216 struct trace_array *tr = m->private; in tracing_clock_show()
7222 i == tr->clock_id ? "[" : "", trace_clocks[i].name, in tracing_clock_show()
7223 i == tr->clock_id ? "]" : ""); in tracing_clock_show()
7238 return -EINVAL; in tracing_set_clock()
7242 tr->clock_id = i; in tracing_set_clock()
7244 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func); in tracing_set_clock()
7250 tracing_reset_online_cpus(&tr->array_buffer); in tracing_set_clock()
7253 if (tr->max_buffer.buffer) in tracing_set_clock()
7254 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func); in tracing_set_clock()
7255 tracing_reset_online_cpus(&tr->max_buffer); in tracing_set_clock()
7266 struct seq_file *m = filp->private_data; in tracing_clock_write()
7267 struct trace_array *tr = m->private; in tracing_clock_write()
7273 return -EINVAL; in tracing_clock_write()
7276 return -EFAULT; in tracing_clock_write()
7293 struct trace_array *tr = inode->i_private; in tracing_clock_open()
7300 ret = single_open(file, tracing_clock_show, inode->i_private); in tracing_clock_open()
7309 struct trace_array *tr = m->private; in tracing_time_stamp_mode_show()
7313 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer)) in tracing_time_stamp_mode_show()
7314 seq_puts(m, "delta [absolute]\n"); in tracing_time_stamp_mode_show()
7316 seq_puts(m, "[delta] absolute\n"); in tracing_time_stamp_mode_show()
7325 struct trace_array *tr = inode->i_private; in tracing_time_stamp_mode_open()
7332 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private); in tracing_time_stamp_mode_open()
7356 if (set && tr->no_filter_buffering_ref++) in tracing_set_filter_buffering()
7360 if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) { in tracing_set_filter_buffering()
7361 ret = -EINVAL; in tracing_set_filter_buffering()
7365 --tr->no_filter_buffering_ref; in tracing_set_filter_buffering()
7384 struct trace_array *tr = inode->i_private; in tracing_snapshot_open()
7393 if (file->f_mode & FMODE_READ) { in tracing_snapshot_open()
7399 ret = -ENOMEM; in tracing_snapshot_open()
7410 iter->tr = tr; in tracing_snapshot_open()
7411 iter->array_buffer = &tr->max_buffer; in tracing_snapshot_open()
7412 iter->cpu_file = tracing_get_cpu(inode); in tracing_snapshot_open()
7413 m->private = iter; in tracing_snapshot_open()
7414 file->private_data = m; in tracing_snapshot_open()
7432 struct seq_file *m = filp->private_data; in tracing_snapshot_write()
7433 struct trace_iterator *iter = m->private; in tracing_snapshot_write()
7434 struct trace_array *tr = iter->tr; in tracing_snapshot_write()
7448 if (tr->current_trace->use_max_tr) { in tracing_snapshot_write()
7449 ret = -EBUSY; in tracing_snapshot_write()
7454 arch_spin_lock(&tr->max_lock); in tracing_snapshot_write()
7455 if (tr->cond_snapshot) in tracing_snapshot_write()
7456 ret = -EBUSY; in tracing_snapshot_write()
7457 arch_spin_unlock(&tr->max_lock); in tracing_snapshot_write()
7464 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { in tracing_snapshot_write()
7465 ret = -EINVAL; in tracing_snapshot_write()
7468 if (tr->allocated_snapshot) in tracing_snapshot_write()
7472 /* Only allow per-cpu swap if the ring buffer supports it */ in tracing_snapshot_write()
7474 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { in tracing_snapshot_write()
7475 ret = -EINVAL; in tracing_snapshot_write()
7479 if (tr->allocated_snapshot) in tracing_snapshot_write()
7480 ret = resize_buffer_duplicate_size(&tr->max_buffer, in tracing_snapshot_write()
7481 &tr->array_buffer, iter->cpu_file); in tracing_snapshot_write()
7488 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { in tracing_snapshot_write()
7493 smp_call_function_single(iter->cpu_file, tracing_swap_cpu_buffer, in tracing_snapshot_write()
7499 if (tr->allocated_snapshot) { in tracing_snapshot_write()
7500 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) in tracing_snapshot_write()
7501 tracing_reset_online_cpus(&tr->max_buffer); in tracing_snapshot_write()
7503 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file); in tracing_snapshot_write()
7519 struct seq_file *m = file->private_data; in tracing_snapshot_release()
7524 if (file->f_mode & FMODE_READ) in tracing_snapshot_release()
7529 kfree(m->private); in tracing_snapshot_release()
7552 info = filp->private_data; in snapshot_raw_open()
7554 if (info->iter.trace->use_max_tr) { in snapshot_raw_open()
7556 return -EBUSY; in snapshot_raw_open()
7559 info->iter.snapshot = true; in snapshot_raw_open()
7560 info->iter.array_buffer = &info->iter.tr->max_buffer; in snapshot_raw_open()
7682 * trace_min_max_write - Write a u64 value to a trace_min_max_param struct
7689 * The filp->private_data must point to a trace_min_max_param structure that
7696 struct trace_min_max_param *param = filp->private_data; in trace_min_max_write()
7701 return -EFAULT; in trace_min_max_write()
7707 if (param->lock) in trace_min_max_write()
7708 mutex_lock(param->lock); in trace_min_max_write()
7710 if (param->min && val < *param->min) in trace_min_max_write()
7711 err = -EINVAL; in trace_min_max_write()
7713 if (param->max && val > *param->max) in trace_min_max_write()
7714 err = -EINVAL; in trace_min_max_write()
7717 *param->val = val; in trace_min_max_write()
7719 if (param->lock) in trace_min_max_write()
7720 mutex_unlock(param->lock); in trace_min_max_write()
7729 * trace_min_max_read - Read a u64 value from a trace_min_max_param struct
7736 * The filp->private_data must point to a trace_min_max_param struct with valid
7742 struct trace_min_max_param *param = filp->private_data; in trace_min_max_read()
7748 return -EFAULT; in trace_min_max_read()
7750 val = *param->val; in trace_min_max_read()
7772 const char **errs; /* ptr to loc-specific array of err strings */
7773 u8 type; /* index into errs -> specific err string */
7793 return ERR_PTR(-ENOMEM); in alloc_tracing_log_err()
7795 err->cmd = kzalloc(len, GFP_KERNEL); in alloc_tracing_log_err()
7796 if (!err->cmd) { in alloc_tracing_log_err()
7798 return ERR_PTR(-ENOMEM); in alloc_tracing_log_err()
7806 kfree(err->cmd); in free_tracing_log_err()
7816 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) { in get_tracing_log_err()
7818 if (PTR_ERR(err) != -ENOMEM) in get_tracing_log_err()
7819 tr->n_err_log_entries++; in get_tracing_log_err()
7825 return ERR_PTR(-ENOMEM); in get_tracing_log_err()
7826 err = list_first_entry(&tr->err_log, struct tracing_log_err, list); in get_tracing_log_err()
7827 kfree(err->cmd); in get_tracing_log_err()
7828 err->cmd = cmd; in get_tracing_log_err()
7829 list_del(&err->list); in get_tracing_log_err()
7835 * err_pos - find the position of a string within a command for error careting
7855 return found - cmd; in err_pos()
7861 * tracing_log_err - write an error to the tracing error log
7865 * @errs: The array of loc-specific static error strings
7882 * produce a static error string - this string is not copied and saved
7883 * when the error is logged - only a pointer to it is saved. See
7901 if (PTR_ERR(err) == -ENOMEM) { in tracing_log_err()
7906 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc); in tracing_log_err()
7907 snprintf(err->cmd, len, "\n" CMD_PREFIX "%s\n", cmd); in tracing_log_err()
7909 err->info.errs = errs; in tracing_log_err()
7910 err->info.type = type; in tracing_log_err()
7911 err->info.pos = pos; in tracing_log_err()
7912 err->info.ts = local_clock(); in tracing_log_err()
7914 list_add_tail(&err->list, &tr->err_log); in tracing_log_err()
7923 list_for_each_entry_safe(err, next, &tr->err_log, list) { in clear_tracing_err_log()
7924 list_del(&err->list); in clear_tracing_err_log()
7928 tr->n_err_log_entries = 0; in clear_tracing_err_log()
7934 struct trace_array *tr = m->private; in tracing_err_log_seq_start()
7938 return seq_list_start(&tr->err_log, *pos); in tracing_err_log_seq_start()
7943 struct trace_array *tr = m->private; in tracing_err_log_seq_next()
7945 return seq_list_next(v, &tr->err_log, pos); in tracing_err_log_seq_next()
7957 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++) in tracing_err_log_show_pos()
7969 const char *err_text = err->info.errs[err->info.type]; in tracing_err_log_seq_show()
7970 u64 sec = err->info.ts; in tracing_err_log_seq_show()
7975 err->loc, err_text); in tracing_err_log_seq_show()
7976 seq_printf(m, "%s", err->cmd); in tracing_err_log_seq_show()
7977 tracing_err_log_show_pos(m, err->info.pos); in tracing_err_log_seq_show()
7992 struct trace_array *tr = inode->i_private; in tracing_err_log_open()
8000 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) in tracing_err_log_open()
8003 if (file->f_mode & FMODE_READ) { in tracing_err_log_open()
8006 struct seq_file *m = file->private_data; in tracing_err_log_open()
8007 m->private = tr; in tracing_err_log_open()
8024 struct trace_array *tr = inode->i_private; in tracing_err_log_release()
8028 if (file->f_mode & FMODE_READ) in tracing_err_log_release()
8044 struct trace_array *tr = inode->i_private; in tracing_buffers_open()
8055 return -ENOMEM; in tracing_buffers_open()
8060 info->iter.tr = tr; in tracing_buffers_open()
8061 info->iter.cpu_file = tracing_get_cpu(inode); in tracing_buffers_open()
8062 info->iter.trace = tr->current_trace; in tracing_buffers_open()
8063 info->iter.array_buffer = &tr->array_buffer; in tracing_buffers_open()
8064 info->spare = NULL; in tracing_buffers_open()
8066 info->read = (unsigned int)-1; in tracing_buffers_open()
8068 filp->private_data = info; in tracing_buffers_open()
8070 tr->trace_ref++; in tracing_buffers_open()
8084 struct ftrace_buffer_info *info = filp->private_data; in tracing_buffers_poll()
8085 struct trace_iterator *iter = &info->iter; in tracing_buffers_poll()
8094 struct ftrace_buffer_info *info = filp->private_data; in tracing_buffers_read()
8095 struct trace_iterator *iter = &info->iter; in tracing_buffers_read()
8105 if (iter->snapshot && iter->tr->current_trace->use_max_tr) in tracing_buffers_read()
8106 return -EBUSY; in tracing_buffers_read()
8109 page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer); in tracing_buffers_read()
8112 if (info->spare) { in tracing_buffers_read()
8113 if (page_size != info->spare_size) { in tracing_buffers_read()
8114 ring_buffer_free_read_page(iter->array_buffer->buffer, in tracing_buffers_read()
8115 info->spare_cpu, info->spare); in tracing_buffers_read()
8116 info->spare = NULL; in tracing_buffers_read()
8120 if (!info->spare) { in tracing_buffers_read()
8121 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer, in tracing_buffers_read()
8122 iter->cpu_file); in tracing_buffers_read()
8123 if (IS_ERR(info->spare)) { in tracing_buffers_read()
8124 ret = PTR_ERR(info->spare); in tracing_buffers_read()
8125 info->spare = NULL; in tracing_buffers_read()
8127 info->spare_cpu = iter->cpu_file; in tracing_buffers_read()
8128 info->spare_size = page_size; in tracing_buffers_read()
8131 if (!info->spare) in tracing_buffers_read()
8135 if (info->read < page_size) in tracing_buffers_read()
8139 trace_access_lock(iter->cpu_file); in tracing_buffers_read()
8140 ret = ring_buffer_read_page(iter->array_buffer->buffer, in tracing_buffers_read()
8141 info->spare, in tracing_buffers_read()
8143 iter->cpu_file, 0); in tracing_buffers_read()
8144 trace_access_unlock(iter->cpu_file); in tracing_buffers_read()
8147 if (trace_empty(iter) && !iter->closed) { in tracing_buffers_read()
8148 if ((filp->f_flags & O_NONBLOCK)) in tracing_buffers_read()
8149 return -EAGAIN; in tracing_buffers_read()
8160 info->read = 0; in tracing_buffers_read()
8162 size = page_size - info->read; in tracing_buffers_read()
8165 trace_data = ring_buffer_read_page_data(info->spare); in tracing_buffers_read()
8166 ret = copy_to_user(ubuf, trace_data + info->read, size); in tracing_buffers_read()
8168 return -EFAULT; in tracing_buffers_read()
8170 size -= ret; in tracing_buffers_read()
8173 info->read += size; in tracing_buffers_read()
8180 struct ftrace_buffer_info *info = file->private_data; in tracing_buffers_flush()
8181 struct trace_iterator *iter = &info->iter; in tracing_buffers_flush()
8183 iter->closed = true; in tracing_buffers_flush()
8185 (void)atomic_fetch_inc_release(&iter->wait_index); in tracing_buffers_flush()
8187 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); in tracing_buffers_flush()
8194 struct ftrace_buffer_info *info = file->private_data; in tracing_buffers_release()
8195 struct trace_iterator *iter = &info->iter; in tracing_buffers_release()
8199 iter->tr->trace_ref--; in tracing_buffers_release()
8201 __trace_array_put(iter->tr); in tracing_buffers_release()
8203 if (info->spare) in tracing_buffers_release()
8204 ring_buffer_free_read_page(iter->array_buffer->buffer, in tracing_buffers_release()
8205 info->spare_cpu, info->spare); in tracing_buffers_release()
8222 if (!refcount_dec_and_test(&ref->refcount)) in buffer_ref_release()
8224 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page); in buffer_ref_release()
8231 struct buffer_ref *ref = (struct buffer_ref *)buf->private; in buffer_pipe_buf_release()
8234 buf->private = 0; in buffer_pipe_buf_release()
8240 struct buffer_ref *ref = (struct buffer_ref *)buf->private; in buffer_pipe_buf_get()
8242 if (refcount_read(&ref->refcount) > INT_MAX/2) in buffer_pipe_buf_get()
8245 refcount_inc(&ref->refcount); in buffer_pipe_buf_get()
8262 (struct buffer_ref *)spd->partial[i].private; in buffer_spd_release()
8265 spd->partial[i].private = 0; in buffer_spd_release()
8273 struct ftrace_buffer_info *info = file->private_data; in tracing_buffers_splice_read()
8274 struct trace_iterator *iter = &info->iter; in tracing_buffers_splice_read()
8291 if (iter->snapshot && iter->tr->current_trace->use_max_tr) in tracing_buffers_splice_read()
8292 return -EBUSY; in tracing_buffers_splice_read()
8295 page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer); in tracing_buffers_splice_read()
8296 if (*ppos & (page_size - 1)) in tracing_buffers_splice_read()
8297 return -EINVAL; in tracing_buffers_splice_read()
8299 if (len & (page_size - 1)) { in tracing_buffers_splice_read()
8301 return -EINVAL; in tracing_buffers_splice_read()
8302 len &= (~(page_size - 1)); in tracing_buffers_splice_read()
8306 return -ENOMEM; in tracing_buffers_splice_read()
8309 trace_access_lock(iter->cpu_file); in tracing_buffers_splice_read()
8310 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); in tracing_buffers_splice_read()
8312 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= page_size) { in tracing_buffers_splice_read()
8318 ret = -ENOMEM; in tracing_buffers_splice_read()
8322 refcount_set(&ref->refcount, 1); in tracing_buffers_splice_read()
8323 ref->buffer = iter->array_buffer->buffer; in tracing_buffers_splice_read()
8324 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file); in tracing_buffers_splice_read()
8325 if (IS_ERR(ref->page)) { in tracing_buffers_splice_read()
8326 ret = PTR_ERR(ref->page); in tracing_buffers_splice_read()
8327 ref->page = NULL; in tracing_buffers_splice_read()
8331 ref->cpu = iter->cpu_file; in tracing_buffers_splice_read()
8333 r = ring_buffer_read_page(ref->buffer, ref->page, in tracing_buffers_splice_read()
8334 len, iter->cpu_file, 1); in tracing_buffers_splice_read()
8336 ring_buffer_free_read_page(ref->buffer, ref->cpu, in tracing_buffers_splice_read()
8337 ref->page); in tracing_buffers_splice_read()
8342 page = virt_to_page(ring_buffer_read_page_data(ref->page)); in tracing_buffers_splice_read()
8351 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); in tracing_buffers_splice_read()
8354 trace_access_unlock(iter->cpu_file); in tracing_buffers_splice_read()
8366 ret = -EAGAIN; in tracing_buffers_splice_read()
8367 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) in tracing_buffers_splice_read()
8370 ret = wait_on_pipe(iter, iter->snapshot ? 0 : iter->tr->buffer_percent); in tracing_buffers_splice_read()
8375 if (!tracer_tracing_is_on(iter->tr)) in tracing_buffers_splice_read()
8393 struct ftrace_buffer_info *info = file->private_data; in tracing_buffers_ioctl()
8394 struct trace_iterator *iter = &info->iter; in tracing_buffers_ioctl()
8398 if (!(file->f_flags & O_NONBLOCK)) { in tracing_buffers_ioctl()
8399 err = ring_buffer_wait(iter->array_buffer->buffer, in tracing_buffers_ioctl()
8400 iter->cpu_file, in tracing_buffers_ioctl()
8401 iter->tr->buffer_percent, in tracing_buffers_ioctl()
8407 return ring_buffer_map_get_reader(iter->array_buffer->buffer, in tracing_buffers_ioctl()
8408 iter->cpu_file); in tracing_buffers_ioctl()
8410 return -ENOTTY; in tracing_buffers_ioctl()
8420 (void)atomic_fetch_inc_release(&iter->wait_index); in tracing_buffers_ioctl()
8422 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); in tracing_buffers_ioctl()
8438 spin_lock(&tr->snapshot_trigger_lock); in get_snapshot_map()
8440 if (tr->snapshot || tr->mapped == UINT_MAX) in get_snapshot_map()
8441 err = -EBUSY; in get_snapshot_map()
8443 tr->mapped++; in get_snapshot_map()
8445 spin_unlock(&tr->snapshot_trigger_lock); in get_snapshot_map()
8447 /* Wait for update_max_tr() to observe iter->tr->mapped */ in get_snapshot_map()
8448 if (tr->mapped == 1) in get_snapshot_map()
8456 spin_lock(&tr->snapshot_trigger_lock); in put_snapshot_map()
8457 if (!WARN_ON(!tr->mapped)) in put_snapshot_map()
8458 tr->mapped--; in put_snapshot_map()
8459 spin_unlock(&tr->snapshot_trigger_lock); in put_snapshot_map()
8468 struct ftrace_buffer_info *info = vma->vm_file->private_data; in tracing_buffers_mmap_close()
8469 struct trace_iterator *iter = &info->iter; in tracing_buffers_mmap_close()
8471 WARN_ON(ring_buffer_unmap(iter->array_buffer->buffer, iter->cpu_file)); in tracing_buffers_mmap_close()
8472 put_snapshot_map(iter->tr); in tracing_buffers_mmap_close()
8481 struct ftrace_buffer_info *info = filp->private_data; in tracing_buffers_mmap()
8482 struct trace_iterator *iter = &info->iter; in tracing_buffers_mmap()
8485 ret = get_snapshot_map(iter->tr); in tracing_buffers_mmap()
8489 ret = ring_buffer_map(iter->array_buffer->buffer, iter->cpu_file, vma); in tracing_buffers_mmap()
8491 put_snapshot_map(iter->tr); in tracing_buffers_mmap()
8493 vma->vm_ops = &tracing_buffers_vmops; in tracing_buffers_mmap()
8514 struct trace_array *tr = inode->i_private; in tracing_stats_read()
8515 struct array_buffer *trace_buf = &tr->array_buffer; in tracing_stats_read()
8524 return -ENOMEM; in tracing_stats_read()
8528 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu); in tracing_stats_read()
8531 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu); in tracing_stats_read()
8534 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu); in tracing_stats_read()
8537 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu); in tracing_stats_read()
8540 if (trace_clocks[tr->clock_id].in_ns) { in tracing_stats_read()
8542 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); in tracing_stats_read()
8547 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer)); in tracing_stats_read()
8553 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); in tracing_stats_read()
8556 ring_buffer_time_stamp(trace_buf->buffer)); in tracing_stats_read()
8559 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu); in tracing_stats_read()
8562 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu); in tracing_stats_read()
8566 s->buffer, trace_seq_used(s)); in tracing_stats_read()
8593 return -ENOMEM; in tracing_read_dyn_info()
8637 (*count)--; in ftrace_count_snapshot()
8674 return -ENOMEM; in ftrace_snapshot_init()
8714 void *count = (void *)-1; in ftrace_trace_snapshot_callback()
8719 return -ENODEV; in ftrace_trace_snapshot_callback()
8723 return -EINVAL; in ftrace_trace_snapshot_callback()
8778 if (WARN_ON(!tr->dir)) in tracing_get_dentry()
8779 return ERR_PTR(-ENODEV); in tracing_get_dentry()
8782 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) in tracing_get_dentry()
8786 return tr->dir; in tracing_get_dentry()
8793 if (tr->percpu_dir) in tracing_dentry_percpu()
8794 return tr->percpu_dir; in tracing_dentry_percpu()
8800 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer); in tracing_dentry_percpu()
8802 MEM_FAIL(!tr->percpu_dir, in tracing_dentry_percpu()
8805 return tr->percpu_dir; in tracing_dentry_percpu()
8815 d_inode(ret)->i_cdev = (void *)(cpu + 1); in trace_create_cpu_file()
8853 if (tr->range_addr_start) in tracing_init_tracefs_percpu()
8857 if (!tr->range_addr_start) { in tracing_init_tracefs_percpu()
8876 struct trace_option_dentry *topt = filp->private_data; in trace_options_read()
8879 if (topt->flags->val & topt->opt->bit) in trace_options_read()
8891 struct trace_option_dentry *topt = filp->private_data; in trace_options_write()
8900 return -EINVAL; in trace_options_write()
8902 if (!!(topt->flags->val & topt->opt->bit) != val) { in trace_options_write()
8904 ret = __set_tracer_option(topt->tr, topt->flags, in trace_options_write()
8905 topt->opt, !val); in trace_options_write()
8918 struct trace_option_dentry *topt = inode->i_private; in tracing_open_options()
8921 ret = tracing_check_open_get_tr(topt->tr); in tracing_open_options()
8925 filp->private_data = inode->i_private; in tracing_open_options()
8931 struct trace_option_dentry *topt = file->private_data; in tracing_release_options()
8933 trace_array_put(topt->tr); in tracing_release_options()
8964 * ptr - idx == &index[0]
8974 *ptr = container_of(data - *pindex, struct trace_array, in get_tr_index()
8982 void *tr_index = filp->private_data; in trace_options_core_read()
8989 if (tr->trace_flags & (1 << index)) in trace_options_core_read()
9001 void *tr_index = filp->private_data; in trace_options_core_write()
9014 return -EINVAL; in trace_options_core_write()
9057 if (tr->options) in trace_options_init_dentry()
9058 return tr->options; in trace_options_init_dentry()
9064 tr->options = tracefs_create_dir("options", d_tracer); in trace_options_init_dentry()
9065 if (!tr->options) { in trace_options_init_dentry()
9070 return tr->options; in trace_options_init_dentry()
9085 topt->flags = flags; in create_trace_option_file()
9086 topt->opt = opt; in create_trace_option_file()
9087 topt->tr = tr; in create_trace_option_file()
9089 topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE, in create_trace_option_file()
9107 flags = tracer->flags; in create_trace_option_files()
9109 if (!flags || !flags->opts) in create_trace_option_files()
9119 for (i = 0; i < tr->nr_topts; i++) { in create_trace_option_files()
9121 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags)) in create_trace_option_files()
9125 opts = flags->opts; in create_trace_option_files()
9134 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1), in create_trace_option_files()
9141 tr->topts = tr_topts; in create_trace_option_files()
9142 tr->topts[tr->nr_topts].tracer = tracer; in create_trace_option_files()
9143 tr->topts[tr->nr_topts].topts = topts; in create_trace_option_files()
9144 tr->nr_topts++; in create_trace_option_files()
9166 (void *)&tr->trace_flags_index[index], in create_trace_option_core_file()
9191 struct trace_array *tr = filp->private_data; in rb_simple_read()
9205 struct trace_array *tr = filp->private_data; in rb_simple_write()
9206 struct trace_buffer *buffer = tr->array_buffer.buffer; in rb_simple_write()
9220 if (tr->current_trace->start) in rb_simple_write()
9221 tr->current_trace->start(tr); in rb_simple_write()
9224 if (tr->current_trace->stop) in rb_simple_write()
9225 tr->current_trace->stop(tr); in rb_simple_write()
9249 struct trace_array *tr = filp->private_data; in buffer_percent_read()
9253 r = tr->buffer_percent; in buffer_percent_read()
9263 struct trace_array *tr = filp->private_data; in buffer_percent_write()
9272 return -EINVAL; in buffer_percent_write()
9274 tr->buffer_percent = val; in buffer_percent_write()
9292 struct trace_array *tr = filp->private_data; in buffer_subbuf_size_read()
9298 order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); in buffer_subbuf_size_read()
9310 struct trace_array *tr = filp->private_data; in buffer_subbuf_size_write()
9324 order = fls(pages - 1); in buffer_subbuf_size_write()
9328 return -EINVAL; in buffer_subbuf_size_write()
9333 old_order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); in buffer_subbuf_size_write()
9337 ret = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, order); in buffer_subbuf_size_write()
9343 if (!tr->allocated_snapshot) in buffer_subbuf_size_write()
9346 ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order); in buffer_subbuf_size_write()
9349 cnt = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, old_order); in buffer_subbuf_size_write()
9396 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0; in allocate_trace_buffer()
9398 buf->tr = tr; in allocate_trace_buffer()
9400 if (tr->range_addr_start && tr->range_addr_size) { in allocate_trace_buffer()
9401 buf->buffer = ring_buffer_alloc_range(size, rb_flags, 0, in allocate_trace_buffer()
9402 tr->range_addr_start, in allocate_trace_buffer()
9403 tr->range_addr_size); in allocate_trace_buffer()
9405 ring_buffer_last_boot_delta(buf->buffer, in allocate_trace_buffer()
9406 &tr->text_delta, &tr->data_delta); in allocate_trace_buffer()
9411 tr->mapped++; in allocate_trace_buffer()
9413 buf->buffer = ring_buffer_alloc(size, rb_flags); in allocate_trace_buffer()
9415 if (!buf->buffer) in allocate_trace_buffer()
9416 return -ENOMEM; in allocate_trace_buffer()
9418 buf->data = alloc_percpu(struct trace_array_cpu); in allocate_trace_buffer()
9419 if (!buf->data) { in allocate_trace_buffer()
9420 ring_buffer_free(buf->buffer); in allocate_trace_buffer()
9421 buf->buffer = NULL; in allocate_trace_buffer()
9422 return -ENOMEM; in allocate_trace_buffer()
9426 set_buffer_entries(&tr->array_buffer, in allocate_trace_buffer()
9427 ring_buffer_size(tr->array_buffer.buffer, 0)); in allocate_trace_buffer()
9434 if (buf->buffer) { in free_trace_buffer()
9435 ring_buffer_free(buf->buffer); in free_trace_buffer()
9436 buf->buffer = NULL; in free_trace_buffer()
9437 free_percpu(buf->data); in free_trace_buffer()
9438 buf->data = NULL; in free_trace_buffer()
9446 ret = allocate_trace_buffer(tr, &tr->array_buffer, size); in allocate_trace_buffers()
9452 if (tr->range_addr_start) in allocate_trace_buffers()
9455 ret = allocate_trace_buffer(tr, &tr->max_buffer, in allocate_trace_buffers()
9458 free_trace_buffer(&tr->array_buffer); in allocate_trace_buffers()
9459 return -ENOMEM; in allocate_trace_buffers()
9461 tr->allocated_snapshot = allocate_snapshot; in allocate_trace_buffers()
9474 free_trace_buffer(&tr->array_buffer); in free_trace_buffers()
9477 free_trace_buffer(&tr->max_buffer); in free_trace_buffers()
9487 tr->trace_flags_index[i] = i; in init_trace_flags_index()
9494 for (t = trace_types; t; t = t->next) in __update_tracer_options()
9512 if (tr->name && strcmp(tr->name, instance) == 0) { in trace_array_find()
9528 tr->ref++; in trace_array_find_get()
9538 tr->dir = tracefs_create_dir(tr->name, trace_instance_dir); in trace_array_create_dir()
9539 if (!tr->dir) in trace_array_create_dir()
9540 return -EINVAL; in trace_array_create_dir()
9542 ret = event_trace_add_tracer(tr->dir, tr); in trace_array_create_dir()
9544 tracefs_remove(tr->dir); in trace_array_create_dir()
9548 init_tracer_tracefs(tr, tr->dir); in trace_array_create_dir()
9562 ret = -ENOMEM; in trace_array_create_systems()
9567 tr->name = kstrdup(name, GFP_KERNEL); in trace_array_create_systems()
9568 if (!tr->name) in trace_array_create_systems()
9571 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL)) in trace_array_create_systems()
9574 if (!zalloc_cpumask_var(&tr->pipe_cpumask, GFP_KERNEL)) in trace_array_create_systems()
9578 tr->system_names = kstrdup_const(systems, GFP_KERNEL); in trace_array_create_systems()
9579 if (!tr->system_names) in trace_array_create_systems()
9584 tr->range_addr_start = range_addr_start; in trace_array_create_systems()
9585 tr->range_addr_size = range_addr_size; in trace_array_create_systems()
9587 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS; in trace_array_create_systems()
9589 cpumask_copy(tr->tracing_cpumask, cpu_all_mask); in trace_array_create_systems()
9591 raw_spin_lock_init(&tr->start_lock); in trace_array_create_systems()
9593 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; in trace_array_create_systems()
9595 spin_lock_init(&tr->snapshot_trigger_lock); in trace_array_create_systems()
9597 tr->current_trace = &nop_trace; in trace_array_create_systems()
9599 INIT_LIST_HEAD(&tr->systems); in trace_array_create_systems()
9600 INIT_LIST_HEAD(&tr->events); in trace_array_create_systems()
9601 INIT_LIST_HEAD(&tr->hist_vars); in trace_array_create_systems()
9602 INIT_LIST_HEAD(&tr->err_log); in trace_array_create_systems()
9624 list_add(&tr->list, &ftrace_trace_arrays); in trace_array_create_systems()
9626 tr->ref++; in trace_array_create_systems()
9633 free_cpumask_var(tr->pipe_cpumask); in trace_array_create_systems()
9634 free_cpumask_var(tr->tracing_cpumask); in trace_array_create_systems()
9635 kfree_const(tr->system_names); in trace_array_create_systems()
9636 kfree(tr->name); in trace_array_create_systems()
9655 ret = -EEXIST; in instance_mkdir()
9695 * trace_array_get_by_name - Create/Lookup a trace array, given its name.
9719 if (tr->name && strcmp(tr->name, name) == 0) in trace_array_get_by_name()
9729 tr->ref++; in trace_array_get_by_name()
9742 if (tr->ref > 1 || (tr->current_trace && tr->trace_ref)) in __remove_instance()
9743 return -EBUSY; in __remove_instance()
9745 list_del(&tr->list); in __remove_instance()
9761 tracefs_remove(tr->dir); in __remove_instance()
9762 free_percpu(tr->last_func_repeats); in __remove_instance()
9766 for (i = 0; i < tr->nr_topts; i++) { in __remove_instance()
9767 kfree(tr->topts[i].topts); in __remove_instance()
9769 kfree(tr->topts); in __remove_instance()
9771 free_cpumask_var(tr->pipe_cpumask); in __remove_instance()
9772 free_cpumask_var(tr->tracing_cpumask); in __remove_instance()
9773 kfree_const(tr->system_names); in __remove_instance()
9774 kfree(tr->name); in __remove_instance()
9786 return -EINVAL; in trace_array_destroy()
9791 ret = -ENODEV; in trace_array_destroy()
9816 ret = -ENODEV; in instance_rmdir()
9841 if (!tr->name) in create_trace_instances()
9887 tr->trace_marker_file = __find_event_file(tr, "ftrace", "print"); in init_tracer_tracefs()
9901 tr->buffer_percent = 50; in init_tracer_tracefs()
9918 if (tr->range_addr_start) { in init_tracer_tracefs()
9960 * tracing_init_dentry - initialize top level trace array
9972 return -EPERM; in tracing_init_dentry()
9976 if (tr->dir) in tracing_init_dentry()
9980 return -ENODEV; in tracing_init_dentry()
9988 tr->dir = debugfs_create_automount("tracing", NULL, in tracing_init_dentry()
10005 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps; in eval_map_work_func()
10018 return -ENOMEM; in trace_eval_init()
10041 if (!mod->num_trace_evals) in trace_module_add_evals()
10051 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals); in trace_module_add_evals()
10060 if (!mod->num_trace_evals) in trace_module_remove_evals()
10068 if (map->head.mod == mod) in trace_module_remove_evals()
10071 last = &map->tail.next; in trace_module_remove_evals()
10072 map = map->tail.next; in trace_module_remove_evals()
10077 *last = trace_eval_jmp_to_tail(map)->tail.next; in trace_module_remove_evals()
10177 .priority = INT_MAX - 1,
10182 .priority = INT_MAX - 1,
10223 if (s->seq.len >= TRACE_MAX_PRINT) in trace_printk_seq()
10224 s->seq.len = TRACE_MAX_PRINT; in trace_printk_seq()
10231 if (WARN_ON_ONCE(s->seq.len >= s->seq.size)) in trace_printk_seq()
10232 s->seq.len = s->seq.size - 1; in trace_printk_seq()
10235 s->buffer[s->seq.len] = 0; in trace_printk_seq()
10237 printk(KERN_TRACE "%s", s->buffer); in trace_printk_seq()
10244 iter->tr = tr; in trace_init_iter()
10245 iter->trace = iter->tr->current_trace; in trace_init_iter()
10246 iter->cpu_file = RING_BUFFER_ALL_CPUS; in trace_init_iter()
10247 iter->array_buffer = &tr->array_buffer; in trace_init_iter()
10249 if (iter->trace && iter->trace->open) in trace_init_iter()
10250 iter->trace->open(iter); in trace_init_iter()
10253 if (ring_buffer_overruns(iter->array_buffer->buffer)) in trace_init_iter()
10254 iter->iter_flags |= TRACE_FILE_ANNOTATE; in trace_init_iter()
10257 if (trace_clocks[iter->tr->clock_id].in_ns) in trace_init_iter()
10258 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; in trace_init_iter()
10261 iter->temp = static_temp_buf; in trace_init_iter()
10262 iter->temp_size = STATIC_TEMP_BUF_SIZE; in trace_init_iter()
10263 iter->fmt = static_fmt_buf; in trace_init_iter()
10264 iter->fmt_size = STATIC_FMT_BUF_SIZE; in trace_init_iter()
10285 * If the user does a sysrq-z, then they can re-enable in ftrace_dump_one()
10296 atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); in ftrace_dump_one()
10299 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ; in ftrace_dump_one()
10302 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ; in ftrace_dump_one()
10312 printk(KERN_TRACE "Dumping ftrace instance %s buffer:\n", tr->name); in ftrace_dump_one()
10330 printk(KERN_TRACE "---------------------------------\n"); in ftrace_dump_one()
10352 printk(KERN_TRACE "---------------------------------\n"); in ftrace_dump_one()
10354 tr->trace_flags |= old_userobj; in ftrace_dump_one()
10357 atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); in ftrace_dump_one()
10447 return -ENOMEM; in trace_parse_run_command()
10450 size = count - done; in trace_parse_run_command()
10453 size = WRITE_BUFSIZE - 1; in trace_parse_run_command()
10456 ret = -EFAULT; in trace_parse_run_command()
10465 size = tmp - buf + 1; in trace_parse_run_command()
10471 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */ in trace_parse_run_command()
10473 WRITE_BUFSIZE - 2); in trace_parse_run_command()
10474 ret = -EINVAL; in trace_parse_run_command()
10551 boot_instance_info[boot_instance_index - 1] = '\0'; in enable_instances()
10653 tr->flags |= TRACE_ARRAY_FL_BOOT; in enable_instances()
10654 tr->ref++; in enable_instances()
10666 int ret = -ENOMEM; in tracer_alloc_buffers()
10671 return -EPERM; in tracer_alloc_buffers()
10714 ret = -ENOMEM; in tracer_alloc_buffers()
10809 if (!tr->allocated_snapshot) in ftrace_boot_snapshot()