// SPDX-License-Identifier: GPL-2.0

/*

* ring buffer based function tracer

*

* Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>

* Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>

*

* Originally taken from the RT patch by:

* Arnaldo Carvalho de Melo <acme@redhat.com>

*

* Based on code from the latency_tracer, that is:

* Copyright (C) 2004-2006 Ingo Molnar

* Copyright (C) 2004 Nadia Yvette Chambers

*/

#include <linux/ring_buffer.h>

#include <generated/utsrelease.h>

#include <linux/stacktrace.h>

#include <linux/writeback.h>

#include <linux/kallsyms.h>

#include <linux/security.h>

#include <linux/seq_file.h>

#include <linux/notifier.h>

#include <linux/irqflags.h>

#include <linux/debugfs.h>

#include <linux/tracefs.h>

#include <linux/pagemap.h>

#include <linux/hardirq.h>

#include <linux/linkage.h>

#include <linux/uaccess.h>

#include <linux/vmalloc.h>

#include <linux/ftrace.h>

#include <linux/module.h>

#include <linux/percpu.h>

#include <linux/splice.h>

#include <linux/kdebug.h>

#include <linux/string.h>

#include <linux/mount.h>

#include <linux/rwsem.h>

#include <linux/slab.h>

#include <linux/ctype.h>

#include <linux/init.h>

#include <linux/panic_notifier.h>

#include <linux/poll.h>

#include <linux/nmi.h>

#include <linux/fs.h>

#include <linux/trace.h>

#include <linux/sched/clock.h>

#include <linux/sched/rt.h>

#include <linux/fsnotify.h>

#include <linux/irq_work.h>

#include <linux/workqueue.h>

#include "trace.h"

#include "trace_output.h"

/*

* On boot up, the ring buffer is set to the minimum size, so that

* we do not waste memory on systems that are not using tracing.

*/

bool ring_buffer_expanded;

/*

* We need to change this state when a selftest is running.

* A selftest will lurk into the ring-buffer to count the

* entries inserted during the selftest although some concurrent

* insertions into the ring-buffer such as trace_printk could occurred

* at the same time, giving false positive or negative results.

*/

static bool __read_mostly tracing_selftest_running;

/*

* If boot-time tracing including tracers/events via kernel cmdline

* is running, we do not want to run SELFTEST.

*/

bool __read_mostly tracing_selftest_disabled;

#ifdef CONFIG_FTRACE_STARTUP_TEST

void __init disable_tracing_selftest(const char *reason)

{

if (!tracing_selftest_disabled) {

tracing_selftest_disabled = true;

pr_info("Ftrace startup test is disabled due to %s\n", reason);

}

}

#endif

/* Pipe tracepoints to printk */

struct trace_iterator *tracepoint_print_iter;

int tracepoint_printk;

static bool tracepoint_printk_stop_on_boot __initdata;

static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);

/* For tracers that don't implement custom flags */

static struct tracer_opt dummy_tracer_opt[] = {

{ }

};

static int

dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)

{

return 0;

}

/*

* To prevent the comm cache from being overwritten when no

* tracing is active, only save the comm when a trace event

* occurred.

*/

static DEFINE_PER_CPU(bool, trace_taskinfo_save);

/*

* Kill all tracing for good (never come back).

* It is initialized to 1 but will turn to zero if the initialization

* of the tracer is successful. But that is the only place that sets

* this back to zero.

*/

static int tracing_disabled = 1;

cpumask_var_t __read_mostly tracing_buffer_mask;

/*

* ftrace_dump_on_oops - variable to dump ftrace buffer on oops

*

* If there is an oops (or kernel panic) and the ftrace_dump_on_oops

* is set, then ftrace_dump is called. This will output the contents

* of the ftrace buffers to the console. This is very useful for

* capturing traces that lead to crashes and outputing it to a

* serial console.

*

* It is default off, but you can enable it with either specifying

* "ftrace_dump_on_oops" in the kernel command line, or setting

* /proc/sys/kernel/ftrace_dump_on_oops

* Set 1 if you want to dump buffers of all CPUs

* Set 2 if you want to dump the buffer of the CPU that triggered oops

*/

enum ftrace_dump_mode ftrace_dump_on_oops;

/* When set, tracing will stop when a WARN*() is hit */

int __disable_trace_on_warning;

#ifdef CONFIG_TRACE_EVAL_MAP_FILE

/* Map of enums to their values, for "eval_map" file */

struct trace_eval_map_head {

struct module *mod;

unsigned long length;

};

union trace_eval_map_item;

struct trace_eval_map_tail {

/*

* "end" is first and points to NULL as it must be different

* than "mod" or "eval_string"

*/

union trace_eval_map_item *next;

const char *end; /* points to NULL */

};

static DEFINE_MUTEX(trace_eval_mutex);

/*

* The trace_eval_maps are saved in an array with two extra elements,

* one at the beginning, and one at the end. The beginning item contains

* the count of the saved maps (head.length), and the module they

* belong to if not built in (head.mod). The ending item contains a

* pointer to the next array of saved eval_map items.

*/

union trace_eval_map_item {

struct trace_eval_map map;

struct trace_eval_map_head head;

struct trace_eval_map_tail tail;

};

static union trace_eval_map_item *trace_eval_maps;

#endif /* CONFIG_TRACE_EVAL_MAP_FILE */

int tracing_set_tracer(struct trace_array *tr, const char *buf);

static void ftrace_trace_userstack(struct trace_array *tr,

struct trace_buffer *buffer,

unsigned int trace_ctx);

#define MAX_TRACER_SIZE 100

static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;

static char *default_bootup_tracer;

static bool allocate_snapshot;

static bool snapshot_at_boot;

static int __init set_cmdline_ftrace(char *str)

{

strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);

default_bootup_tracer = bootup_tracer_buf;

/* We are using ftrace early, expand it */

ring_buffer_expanded = true;

return 1;

}

__setup("ftrace=", set_cmdline_ftrace);

static int __init set_ftrace_dump_on_oops(char *str)

{

if (*str++ != '=' || !*str || !strcmp("1", str)) {

ftrace_dump_on_oops = DUMP_ALL;

return 1;

}

if (!strcmp("orig_cpu", str) || !strcmp("2", str)) {

ftrace_dump_on_oops = DUMP_ORIG;

return 1;

}

return 0;

}

__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);

static int __init stop_trace_on_warning(char *str)

{

if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))

__disable_trace_on_warning = 1;

return 1;

}

__setup("traceoff_on_warning", stop_trace_on_warning);

static int __init boot_alloc_snapshot(char *str)

{

allocate_snapshot = true;

/* We also need the main ring buffer expanded */

ring_buffer_expanded = true;

return 1;

}

__setup("alloc_snapshot", boot_alloc_snapshot);

static int __init boot_snapshot(char *str)

{

snapshot_at_boot = true;

boot_alloc_snapshot(str);

return 1;

}

__setup("ftrace_boot_snapshot", boot_snapshot);

static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;

static int __init set_trace_boot_options(char *str)

{

strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);

return 1;

}

__setup("trace_options=", set_trace_boot_options);

static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;

static char *trace_boot_clock __initdata;

static int __init set_trace_boot_clock(char *str)

{

strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);

trace_boot_clock = trace_boot_clock_buf;

return 1;

}

__setup("trace_clock=", set_trace_boot_clock);

static int __init set_tracepoint_printk(char *str)

{

/* Ignore the "tp_printk_stop_on_boot" param */

if (*str == '_')

return 0;

if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))

tracepoint_printk = 1;

return 1;

}

__setup("tp_printk", set_tracepoint_printk);

static int __init set_tracepoint_printk_stop(char *str)

{

tracepoint_printk_stop_on_boot = true;

return 1;

}

__setup("tp_printk_stop_on_boot", set_tracepoint_printk_stop);

unsigned long long ns2usecs(u64 nsec)

{

nsec += 500;

do_div(nsec, 1000);

return nsec;

}

static void

trace_process_export(struct trace_export *export,

struct ring_buffer_event *event, int flag)

{

struct trace_entry *entry;

unsigned int size = 0;

if (export->flags & flag) {

entry = ring_buffer_event_data(event);

size = ring_buffer_event_length(event);

export->write(export, entry, size);

}

}

static DEFINE_MUTEX(ftrace_export_lock);

static struct trace_export __rcu *ftrace_exports_list __read_mostly;

static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled);

static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled);

static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled);

static inline void ftrace_exports_enable(struct trace_export *export)

{

if (export->flags & TRACE_EXPORT_FUNCTION)

static_branch_inc(&trace_function_exports_enabled);

if (export->flags & TRACE_EXPORT_EVENT)

static_branch_inc(&trace_event_exports_enabled);

if (export->flags & TRACE_EXPORT_MARKER)

static_branch_inc(&trace_marker_exports_enabled);

}

static inline void ftrace_exports_disable(struct trace_export *export)

{

if (export->flags & TRACE_EXPORT_FUNCTION)

static_branch_dec(&trace_function_exports_enabled);

if (export->flags & TRACE_EXPORT_EVENT)

static_branch_dec(&trace_event_exports_enabled);

if (export->flags & TRACE_EXPORT_MARKER)

static_branch_dec(&trace_marker_exports_enabled);

}

static void ftrace_exports(struct ring_buffer_event *event, int flag)

{

struct trace_export *export;

preempt_disable_notrace();

export = rcu_dereference_raw_check(ftrace_exports_list);

while (export) {

trace_process_export(export, event, flag);

export = rcu_dereference_raw_check(export->next);

}

preempt_enable_notrace();

}

static inline void

add_trace_export(struct trace_export **list, struct trace_export *export)

{

rcu_assign_pointer(export->next, *list);

/*

* We are entering export into the list but another

* CPU might be walking that list. We need to make sure

* the export->next pointer is valid before another CPU sees

* the export pointer included into the list.

*/

rcu_assign_pointer(*list, export);

}

static inline int

rm_trace_export(struct trace_export **list, struct trace_export *export)

{

struct trace_export **p;

for (p = list; *p != NULL; p = &(*p)->next)

if (*p == export)

break;

if (*p != export)

return -1;

rcu_assign_pointer(*p, (*p)->next);

return 0;

}

static inline void

add_ftrace_export(struct trace_export **list, struct trace_export *export)

{

ftrace_exports_enable(export);

add_trace_export(list, export);

}

static inline int

rm_ftrace_export(struct trace_export **list, struct trace_export *export)

{

int ret;

ret = rm_trace_export(list, export);

ftrace_exports_disable(export);

return ret;

}

int register_ftrace_export(struct trace_export *export)

{

if (WARN_ON_ONCE(!export->write))

return -1;

mutex_lock(&ftrace_export_lock);

add_ftrace_export(&ftrace_exports_list, export);

mutex_unlock(&ftrace_export_lock);

return 0;

}

EXPORT_SYMBOL_GPL(register_ftrace_export);

int unregister_ftrace_export(struct trace_export *export)

{

int ret;

mutex_lock(&ftrace_export_lock);

ret = rm_ftrace_export(&ftrace_exports_list, export);

mutex_unlock(&ftrace_export_lock);

return ret;

}

EXPORT_SYMBOL_GPL(unregister_ftrace_export);

/* trace_flags holds trace_options default values */

#define TRACE_DEFAULT_FLAGS \

(FUNCTION_DEFAULT_FLAGS | \

TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \

TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \

TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \

TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | \

TRACE_ITER_HASH_PTR)

/* trace_options that are only supported by global_trace */

#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \

TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)

/* trace_flags that are default zero for instances */

#define ZEROED_TRACE_FLAGS \

(TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)

/*

* The global_trace is the descriptor that holds the top-level tracing

* buffers for the live tracing.

*/

static struct trace_array global_trace = {

.trace_flags = TRACE_DEFAULT_FLAGS,

};

LIST_HEAD(ftrace_trace_arrays);

int trace_array_get(struct trace_array *this_tr)

{

struct trace_array *tr;

int ret = -ENODEV;

mutex_lock(&trace_types_lock);

list_for_each_entry(tr, &ftrace_trace_arrays, list) {

if (tr == this_tr) {

tr->ref++;

ret = 0;

break;

}

}

mutex_unlock(&trace_types_lock);

return ret;

}

static void __trace_array_put(struct trace_array *this_tr)

{

WARN_ON(!this_tr->ref);

this_tr->ref--;

}

/**

* trace_array_put - Decrement the reference counter for this trace array.

* @this_tr : pointer to the trace array

*

* NOTE: Use this when we no longer need the trace array returned by

* trace_array_get_by_name(). This ensures the trace array can be later

* destroyed.

*

*/

void trace_array_put(struct trace_array *this_tr)

{

if (!this_tr)

return;

mutex_lock(&trace_types_lock);

__trace_array_put(this_tr);

mutex_unlock(&trace_types_lock);

}

EXPORT_SYMBOL_GPL(trace_array_put);

int tracing_check_open_get_tr(struct trace_array *tr)

{

int ret;

ret = security_locked_down(LOCKDOWN_TRACEFS);

if (ret)

return ret;

if (tracing_disabled)

return -ENODEV;

if (tr && trace_array_get(tr) < 0)

return -ENODEV;

return 0;

}

int call_filter_check_discard(struct trace_event_call *call, void *rec,

struct trace_buffer *buffer,

struct ring_buffer_event *event)

{

if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&

!filter_match_preds(call->filter, rec)) {

__trace_event_discard_commit(buffer, event);

return 1;

}

return 0;

}

/**

* trace_find_filtered_pid - check if a pid exists in a filtered_pid list

* @filtered_pids: The list of pids to check

* @search_pid: The PID to find in @filtered_pids

*

* Returns true if @search_pid is found in @filtered_pids, and false otherwise.

*/

bool

trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)

{

return trace_pid_list_is_set(filtered_pids, search_pid);

}

/**

* trace_ignore_this_task - should a task be ignored for tracing

* @filtered_pids: The list of pids to check

* @filtered_no_pids: The list of pids not to be traced

* @task: The task that should be ignored if not filtered

*

* Checks if @task should be traced or not from @filtered_pids.

* Returns true if @task should *NOT* be traced.

* Returns false if @task should be traced.

*/

bool

trace_ignore_this_task(struct trace_pid_list *filtered_pids,

struct trace_pid_list *filtered_no_pids,

struct task_struct *task)

{

/*

* If filtered_no_pids is not empty, and the task's pid is listed

* in filtered_no_pids, then return true.

* Otherwise, if filtered_pids is empty, that means we can

* trace all tasks. If it has content, then only trace pids

* within filtered_pids.

*/

return (filtered_pids &&

!trace_find_filtered_pid(filtered_pids, task->pid)) ||

(filtered_no_pids &&

trace_find_filtered_pid(filtered_no_pids, task->pid));

}

/**

* trace_filter_add_remove_task - Add or remove a task from a pid_list

* @pid_list: The list to modify

* @self: The current task for fork or NULL for exit

* @task: The task to add or remove

*

* If adding a task, if @self is defined, the task is only added if @self

* is also included in @pid_list. This happens on fork and tasks should

* only be added when the parent is listed. If @self is NULL, then the

* @task pid will be removed from the list, which would happen on exit

* of a task.

*/

void trace_filter_add_remove_task(struct trace_pid_list *pid_list,

struct task_struct *self,

struct task_struct *task)

{

if (!pid_list)

return;

/* For forks, we only add if the forking task is listed */

if (self) {

if (!trace_find_filtered_pid(pid_list, self->pid))

return;

}

/* "self" is set for forks, and NULL for exits */

if (self)

trace_pid_list_set(pid_list, task->pid);

else

trace_pid_list_clear(pid_list, task->pid);

}

/**

* trace_pid_next - Used for seq_file to get to the next pid of a pid_list

* @pid_list: The pid list to show

* @v: The last pid that was shown (+1 the actual pid to let zero be displayed)

* @pos: The position of the file

*

* This is used by the seq_file "next" operation to iterate the pids

* listed in a trace_pid_list structure.

*

* Returns the pid+1 as we want to display pid of zero, but NULL would

* stop the iteration.

*/

void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)

{

long pid = (unsigned long)v;

unsigned int next;

(*pos)++;

/* pid already is +1 of the actual previous bit */

if (trace_pid_list_next(pid_list, pid, &next) < 0)

return NULL;

pid = next;

/* Return pid + 1 to allow zero to be represented */

return (void *)(pid + 1);

}

/**

* trace_pid_start - Used for seq_file to start reading pid lists

* @pid_list: The pid list to show

* @pos: The position of the file

*

* This is used by seq_file "start" operation to start the iteration

* of listing pids.

*

* Returns the pid+1 as we want to display pid of zero, but NULL would

* stop the iteration.

*/

void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)

{

unsigned long pid;

unsigned int first;

loff_t l = 0;

if (trace_pid_list_first(pid_list, &first) < 0)

return NULL;

pid = first;

/* Return pid + 1 so that zero can be the exit value */

for (pid++; pid && l < *pos;

pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))

;

return (void *)pid;

}

/**

* trace_pid_show - show the current pid in seq_file processing

* @m: The seq_file structure to write into

* @v: A void pointer of the pid (+1) value to display

*

* Can be directly used by seq_file operations to display the current

* pid value.

*/

int trace_pid_show(struct seq_file *m, void *v)

{

unsigned long pid = (unsigned long)v - 1;

seq_printf(m, "%lu\n", pid);

return 0;

}

/* 128 should be much more than enough */

#define PID_BUF_SIZE 127

int trace_pid_write(struct trace_pid_list *filtered_pids,

struct trace_pid_list **new_pid_list,

const char __user *ubuf, size_t cnt)

{

struct trace_pid_list *pid_list;

struct trace_parser parser;

unsigned long val;

int nr_pids = 0;

ssize_t read = 0;

ssize_t ret;

loff_t pos;

pid_t pid;

if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))

return -ENOMEM;

/*

* Always recreate a new array. The write is an all or nothing

* operation. Always create a new array when adding new pids by

* the user. If the operation fails, then the current list is

* not modified.

*/

pid_list = trace_pid_list_alloc();

if (!pid_list) {

trace_parser_put(&parser);

return -ENOMEM;

}

if (filtered_pids) {

/* copy the current bits to the new max */

ret = trace_pid_list_first(filtered_pids, &pid);

while (!ret) {

trace_pid_list_set(pid_list, pid);

ret = trace_pid_list_next(filtered_pids, pid + 1, &pid);

nr_pids++;

}

}

ret = 0;

while (cnt > 0) {

pos = 0;

ret = trace_get_user(&parser, ubuf, cnt, &pos);

if (ret < 0 || !trace_parser_loaded(&parser))

break;

read += ret;

ubuf += ret;

cnt -= ret;

ret = -EINVAL;

if (kstrtoul(parser.buffer, 0, &val))

break;

pid = (pid_t)val;

if (trace_pid_list_set(pid_list, pid) < 0) {

ret = -1;

break;

}

nr_pids++;

trace_parser_clear(&parser);

ret = 0;

}

trace_parser_put(&parser);

if (ret < 0) {

trace_pid_list_free(pid_list);

return ret;

}

if (!nr_pids) {

/* Cleared the list of pids */

trace_pid_list_free(pid_list);

read = ret;

pid_list = NULL;

}

*new_pid_list = pid_list;

return read;

}

static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)

{

u64 ts;

/* Early boot up does not have a buffer yet */

if (!buf->buffer)

return trace_clock_local();

ts = ring_buffer_time_stamp(buf->buffer);

ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);

return ts;

}

u64 ftrace_now(int cpu)

{

return buffer_ftrace_now(&global_trace.array_buffer, cpu);

}

/**

* tracing_is_enabled - Show if global_trace has been enabled

*

* Shows if the global trace has been enabled or not. It uses the

* mirror flag "buffer_disabled" to be used in fast paths such as for

* the irqsoff tracer. But it may be inaccurate due to races. If you

* need to know the accurate state, use tracing_is_on() which is a little

* slower, but accurate.

*/

int tracing_is_enabled(void)

{

/*

* For quick access (irqsoff uses this in fast path), just

* return the mirror variable of the state of the ring buffer.

* It's a little racy, but we don't really care.

*/

smp_rmb();

return !global_trace.buffer_disabled;

}

/*

* trace_buf_size is the size in bytes that is allocated

* for a buffer. Note, the number of bytes is always rounded

* to page size.

*

* This number is purposely set to a low number of 16384.

* If the dump on oops happens, it will be much appreciated

* to not have to wait for all that output. Anyway this can be

* boot time and run time configurable.

*/

#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */

static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;

/* trace_types holds a link list of available tracers. */

static struct tracer *trace_types __read_mostly;

/*

* trace_types_lock is used to protect the trace_types list.

*/

DEFINE_MUTEX(trace_types_lock);

/*

* serialize the access of the ring buffer

*

* ring buffer serializes readers, but it is low level protection.

* The validity of the events (which returns by ring_buffer_peek() ..etc)

* are not protected by ring buffer.

*

* The content of events may become garbage if we allow other process consumes

* these events concurrently:

* A) the page of the consumed events may become a normal page

* (not reader page) in ring buffer, and this page will be rewritten

* by events producer.

* B) The page of the consumed events may become a page for splice_read,

* and this page will be returned to system.

*

* These primitives allow multi process access to different cpu ring buffer

* concurrently.

*

* These primitives don't distinguish read-only and read-consume access.

* Multi read-only access are also serialized.

*/

#ifdef CONFIG_SMP

static DECLARE_RWSEM(all_cpu_access_lock);

static DEFINE_PER_CPU(struct mutex, cpu_access_lock);

static inline void trace_access_lock(int cpu)

{

if (cpu == RING_BUFFER_ALL_CPUS) {

/* gain it for accessing the whole ring buffer. */

down_write(&all_cpu_access_lock);

} else {

/* gain it for accessing a cpu ring buffer. */

/* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */

down_read(&all_cpu_access_lock);

/* Secondly block other access to this @cpu ring buffer. */

mutex_lock(&per_cpu(cpu_access_lock, cpu));

}

}

static inline void trace_access_unlock(int cpu)

{

if (cpu == RING_BUFFER_ALL_CPUS) {

up_write(&all_cpu_access_lock);

} else {

mutex_unlock(&per_cpu(cpu_access_lock, cpu));

up_read(&all_cpu_access_lock);

}

}

static inline void trace_access_lock_init(void)

{

int cpu;

for_each_possible_cpu(cpu)

mutex_init(&per_cpu(cpu_access_lock, cpu));

}

#else

static DEFINE_MUTEX(access_lock);

static inline void trace_access_lock(int cpu)

{

(void)cpu;

mutex_lock(&access_lock);

}

static inline void trace_access_unlock(int cpu)

{

(void)cpu;

mutex_unlock(&access_lock);

}

static inline void trace_access_lock_init(void)

{

}

#endif

#ifdef CONFIG_STACKTRACE

static void __ftrace_trace_stack(struct trace_buffer *buffer,

unsigned int trace_ctx,

int skip, struct pt_regs *regs);

static inline void ftrace_trace_stack(struct trace_array *tr,

struct trace_buffer *buffer,

unsigned int trace_ctx,

int skip, struct pt_regs *regs);

#else

static inline void __ftrace_trace_stack(struct trace_buffer *buffer,

unsigned int trace_ctx,

int skip, struct pt_regs *regs)

{

}

static inline void ftrace_trace_stack(struct trace_array *tr,

struct trace_buffer *buffer,

unsigned long trace_ctx,

int skip, struct pt_regs *regs)

{

}

#endif

static __always_inline void

trace_event_setup(struct ring_buffer_event *event,

int type, unsigned int trace_ctx)

{

struct trace_entry *ent = ring_buffer_event_data(event);

tracing_generic_entry_update(ent, type, trace_ctx);

}

static __always_inline struct ring_buffer_event *

__trace_buffer_lock_reserve(struct trace_buffer *buffer,

int type,

unsigned long len,

unsigned int trace_ctx)

{

struct ring_buffer_event *event;

event = ring_buffer_lock_reserve(buffer, len);

if (event != NULL)

trace_event_setup(event, type, trace_ctx);

return event;

}

void tracer_tracing_on(struct trace_array *tr)

{

if (tr->array_buffer.buffer)

ring_buffer_record_on(tr->array_buffer.buffer);

/*

* This flag is looked at when buffers haven't been allocated

* yet, or by some tracers (like irqsoff), that just want to

* know if the ring buffer has been disabled, but it can handle

* races of where it gets disabled but we still do a record.

* As the check is in the fast path of the tracers, it is more

* important to be fast than accurate.

*/

tr->buffer_disabled = 0;

/* Make the flag seen by readers */

smp_wmb();

}

/**

* tracing_on - enable tracing buffers

*

* This function enables tracing buffers that may have been

* disabled with tracing_off.

*/

void tracing_on(void)

{

tracer_tracing_on(&global_trace);

}

EXPORT_SYMBOL_GPL(tracing_on);

static __always_inline void

__buffer_unlock_commit(struct trace_buffer *buffer, struct ring_buffer_event *event)

{

__this_cpu_write(trace_taskinfo_save, true);

/* If this is the temp buffer, we need to commit fully */

if (this_cpu_read(trace_buffered_event) == event) {

/* Length is in event->array[0] */

ring_buffer_write(buffer, event->array[0], &event->array[1]);

/* Release the temp buffer */

this_cpu_dec(trace_buffered_event_cnt);

/* ring_buffer_unlock_commit() enables preemption */

preempt_enable_notrace();

} else

ring_buffer_unlock_commit(buffer, event);