/*

* ring buffer based function tracer

*

* Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>

* Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>

*

* Originally taken from the RT patch by:

* Arnaldo Carvalho de Melo <acme@redhat.com>

*

* Based on code from the latency_tracer, that is:

* Copyright (C) 2004-2006 Ingo Molnar

* Copyright (C) 2004 Nadia Yvette Chambers

*/

#include <linux/ring_buffer.h>

#include <generated/utsrelease.h>

#include <linux/stacktrace.h>

#include <linux/writeback.h>

#include <linux/kallsyms.h>

#include <linux/seq_file.h>

#include <linux/notifier.h>

#include <linux/irqflags.h>

#include <linux/debugfs.h>

#include <linux/pagemap.h>

#include <linux/hardirq.h>

#include <linux/linkage.h>

#include <linux/uaccess.h>

#include <linux/kprobes.h>

#include <linux/ftrace.h>

#include <linux/module.h>

#include <linux/percpu.h>

#include <linux/splice.h>

#include <linux/kdebug.h>

#include <linux/string.h>

#include <linux/rwsem.h>

#include <linux/slab.h>

#include <linux/ctype.h>

#include <linux/init.h>

#include <linux/poll.h>

#include <linux/nmi.h>

#include <linux/fs.h>

#include <linux/sched/rt.h>

#include "trace.h"

#include "trace_output.h"

/*

* On boot up, the ring buffer is set to the minimum size, so that

* we do not waste memory on systems that are not using tracing.

*/

bool ring_buffer_expanded;

/*

* We need to change this state when a selftest is running.

* A selftest will lurk into the ring-buffer to count the

* entries inserted during the selftest although some concurrent

* insertions into the ring-buffer such as trace_printk could occurred

* at the same time, giving false positive or negative results.

*/

static bool __read_mostly tracing_selftest_running;

/*

* If a tracer is running, we do not want to run SELFTEST.

*/

bool __read_mostly tracing_selftest_disabled;

/* For tracers that don't implement custom flags */

static struct tracer_opt dummy_tracer_opt[] = {

{ }

};

static struct tracer_flags dummy_tracer_flags = {

.val = 0,

.opts = dummy_tracer_opt

};

static int dummy_set_flag(u32 old_flags, u32 bit, int set)

{

return 0;

}

/*

* To prevent the comm cache from being overwritten when no

* tracing is active, only save the comm when a trace event

* occurred.

*/

static DEFINE_PER_CPU(bool, trace_cmdline_save);

/*

* Kill all tracing for good (never come back).

* It is initialized to 1 but will turn to zero if the initialization

* of the tracer is successful. But that is the only place that sets

* this back to zero.

*/

static int tracing_disabled = 1;

DEFINE_PER_CPU(int, ftrace_cpu_disabled);

cpumask_var_t __read_mostly tracing_buffer_mask;

/*

* ftrace_dump_on_oops - variable to dump ftrace buffer on oops

*

* If there is an oops (or kernel panic) and the ftrace_dump_on_oops

* is set, then ftrace_dump is called. This will output the contents

* of the ftrace buffers to the console. This is very useful for

* capturing traces that lead to crashes and outputing it to a

* serial console.

*

* It is default off, but you can enable it with either specifying

* "ftrace_dump_on_oops" in the kernel command line, or setting

* /proc/sys/kernel/ftrace_dump_on_oops

* Set 1 if you want to dump buffers of all CPUs

* Set 2 if you want to dump the buffer of the CPU that triggered oops

*/

enum ftrace_dump_mode ftrace_dump_on_oops;

/* When set, tracing will stop when a WARN*() is hit */

int __disable_trace_on_warning;

static int tracing_set_tracer(const char *buf);

#define MAX_TRACER_SIZE 100

static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;

static char *default_bootup_tracer;

static bool allocate_snapshot;

static int __init set_cmdline_ftrace(char *str)

{

strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);

default_bootup_tracer = bootup_tracer_buf;

/* We are using ftrace early, expand it */

ring_buffer_expanded = true;

return 1;

}

__setup("ftrace=", set_cmdline_ftrace);

static int __init set_ftrace_dump_on_oops(char *str)

{

if (*str++ != '=' || !*str) {

ftrace_dump_on_oops = DUMP_ALL;

return 1;

}

if (!strcmp("orig_cpu", str)) {

ftrace_dump_on_oops = DUMP_ORIG;

return 1;

}

return 0;

}

__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);

static int __init stop_trace_on_warning(char *str)

{

__disable_trace_on_warning = 1;

return 1;

}

__setup("traceoff_on_warning=", stop_trace_on_warning);

static int __init boot_alloc_snapshot(char *str)

{

allocate_snapshot = true;

/* We also need the main ring buffer expanded */

ring_buffer_expanded = true;

return 1;

}

__setup("alloc_snapshot", boot_alloc_snapshot);

static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;

static char *trace_boot_options __initdata;

static int __init set_trace_boot_options(char *str)

{

strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);

trace_boot_options = trace_boot_options_buf;

return 0;

}

__setup("trace_options=", set_trace_boot_options);

unsigned long long ns2usecs(cycle_t nsec)

{

nsec += 500;

do_div(nsec, 1000);

return nsec;

}

/*

* The global_trace is the descriptor that holds the tracing

* buffers for the live tracing. For each CPU, it contains

* a link list of pages that will store trace entries. The

* page descriptor of the pages in the memory is used to hold

* the link list by linking the lru item in the page descriptor

* to each of the pages in the buffer per CPU.

*

* For each active CPU there is a data field that holds the

* pages for the buffer for that CPU. Each CPU has the same number

* of pages allocated for its buffer.

*/

static struct trace_array global_trace;

LIST_HEAD(ftrace_trace_arrays);

int trace_array_get(struct trace_array *this_tr)

{

struct trace_array *tr;

int ret = -ENODEV;

mutex_lock(&trace_types_lock);

list_for_each_entry(tr, &ftrace_trace_arrays, list) {

if (tr == this_tr) {

tr->ref++;

ret = 0;

break;

}

}

mutex_unlock(&trace_types_lock);

return ret;

}

static void __trace_array_put(struct trace_array *this_tr)

{

WARN_ON(!this_tr->ref);

this_tr->ref--;

}

void trace_array_put(struct trace_array *this_tr)

{

mutex_lock(&trace_types_lock);

__trace_array_put(this_tr);

mutex_unlock(&trace_types_lock);

}

int filter_check_discard(struct ftrace_event_file *file, void *rec,

struct ring_buffer *buffer,

struct ring_buffer_event *event)

{

if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&

!filter_match_preds(file->filter, rec)) {

ring_buffer_discard_commit(buffer, event);

return 1;

}

return 0;

}

EXPORT_SYMBOL_GPL(filter_check_discard);

int call_filter_check_discard(struct ftrace_event_call *call, void *rec,

struct ring_buffer *buffer,

struct ring_buffer_event *event)

{

if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&

!filter_match_preds(call->filter, rec)) {

ring_buffer_discard_commit(buffer, event);

return 1;

}

return 0;

}

EXPORT_SYMBOL_GPL(call_filter_check_discard);

cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)

{

u64 ts;

/* Early boot up does not have a buffer yet */

if (!buf->buffer)

return trace_clock_local();

ts = ring_buffer_time_stamp(buf->buffer, cpu);

ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);

return ts;

}

cycle_t ftrace_now(int cpu)

{

return buffer_ftrace_now(&global_trace.trace_buffer, cpu);

}

/**

* tracing_is_enabled - Show if global_trace has been disabled

*

* Shows if the global trace has been enabled or not. It uses the

* mirror flag "buffer_disabled" to be used in fast paths such as for

* the irqsoff tracer. But it may be inaccurate due to races. If you

* need to know the accurate state, use tracing_is_on() which is a little

* slower, but accurate.

*/

int tracing_is_enabled(void)

{

/*

* For quick access (irqsoff uses this in fast path), just

* return the mirror variable of the state of the ring buffer.

* It's a little racy, but we don't really care.

*/

smp_rmb();

return !global_trace.buffer_disabled;

}

/*

* trace_buf_size is the size in bytes that is allocated

* for a buffer. Note, the number of bytes is always rounded

* to page size.

*

* This number is purposely set to a low number of 16384.

* If the dump on oops happens, it will be much appreciated

* to not have to wait for all that output. Anyway this can be

* boot time and run time configurable.

*/

#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */

static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;

/* trace_types holds a link list of available tracers. */

static struct tracer *trace_types __read_mostly;

/*

* trace_types_lock is used to protect the trace_types list.

*/

DEFINE_MUTEX(trace_types_lock);

/*

* serialize the access of the ring buffer

*

* ring buffer serializes readers, but it is low level protection.

* The validity of the events (which returns by ring_buffer_peek() ..etc)

* are not protected by ring buffer.

*

* The content of events may become garbage if we allow other process consumes

* these events concurrently:

* A) the page of the consumed events may become a normal page

* (not reader page) in ring buffer, and this page will be rewrited

* by events producer.

* B) The page of the consumed events may become a page for splice_read,

* and this page will be returned to system.

*

* These primitives allow multi process access to different cpu ring buffer

* concurrently.

*

* These primitives don't distinguish read-only and read-consume access.

* Multi read-only access are also serialized.

*/

#ifdef CONFIG_SMP

static DECLARE_RWSEM(all_cpu_access_lock);

static DEFINE_PER_CPU(struct mutex, cpu_access_lock);

static inline void trace_access_lock(int cpu)

{

if (cpu == RING_BUFFER_ALL_CPUS) {

/* gain it for accessing the whole ring buffer. */

down_write(&all_cpu_access_lock);

} else {

/* gain it for accessing a cpu ring buffer. */

/* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */

down_read(&all_cpu_access_lock);

/* Secondly block other access to this @cpu ring buffer. */

mutex_lock(&per_cpu(cpu_access_lock, cpu));

}

}

static inline void trace_access_unlock(int cpu)

{

if (cpu == RING_BUFFER_ALL_CPUS) {

up_write(&all_cpu_access_lock);

} else {

mutex_unlock(&per_cpu(cpu_access_lock, cpu));

up_read(&all_cpu_access_lock);

}

}

static inline void trace_access_lock_init(void)

{

int cpu;

for_each_possible_cpu(cpu)

mutex_init(&per_cpu(cpu_access_lock, cpu));

}

#else

static DEFINE_MUTEX(access_lock);

static inline void trace_access_lock(int cpu)

{

(void)cpu;

mutex_lock(&access_lock);

}

static inline void trace_access_unlock(int cpu)

{

(void)cpu;

mutex_unlock(&access_lock);

}

static inline void trace_access_lock_init(void)

{

}

#endif

/* trace_flags holds trace_options default values */

unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |

TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |

TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |

TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;

static void tracer_tracing_on(struct trace_array *tr)

{

if (tr->trace_buffer.buffer)

ring_buffer_record_on(tr->trace_buffer.buffer);

/*

* This flag is looked at when buffers haven't been allocated

* yet, or by some tracers (like irqsoff), that just want to

* know if the ring buffer has been disabled, but it can handle

* races of where it gets disabled but we still do a record.

* As the check is in the fast path of the tracers, it is more

* important to be fast than accurate.

*/

tr->buffer_disabled = 0;

/* Make the flag seen by readers */

smp_wmb();

}

/**

* tracing_on - enable tracing buffers

*

* This function enables tracing buffers that may have been

* disabled with tracing_off.

*/

void tracing_on(void)

{

tracer_tracing_on(&global_trace);

}

EXPORT_SYMBOL_GPL(tracing_on);

/**

* __trace_puts - write a constant string into the trace buffer.

* @ip: The address of the caller

* @str: The constant string to write

* @size: The size of the string.

*/

int __trace_puts(unsigned long ip, const char *str, int size)

{

struct ring_buffer_event *event;

struct ring_buffer *buffer;

struct print_entry *entry;

unsigned long irq_flags;

int alloc;

alloc = sizeof(*entry) + size + 2; /* possible \n added */

local_save_flags(irq_flags);

buffer = global_trace.trace_buffer.buffer;

event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,

irq_flags, preempt_count());

if (!event)

return 0;

entry = ring_buffer_event_data(event);

entry->ip = ip;

memcpy(&entry->buf, str, size);

/* Add a newline if necessary */

if (entry->buf[size - 1] != '\n') {

entry->buf[size] = '\n';

entry->buf[size + 1] = '\0';

} else

entry->buf[size] = '\0';

__buffer_unlock_commit(buffer, event);

return size;

}

EXPORT_SYMBOL_GPL(__trace_puts);

/**

* __trace_bputs - write the pointer to a constant string into trace buffer

* @ip: The address of the caller

* @str: The constant string to write to the buffer to

*/

int __trace_bputs(unsigned long ip, const char *str)

{

struct ring_buffer_event *event;

struct ring_buffer *buffer;

struct bputs_entry *entry;

unsigned long irq_flags;

int size = sizeof(struct bputs_entry);

local_save_flags(irq_flags);

buffer = global_trace.trace_buffer.buffer;

event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,

irq_flags, preempt_count());

if (!event)

return 0;

entry = ring_buffer_event_data(event);

entry->ip = ip;

entry->str = str;

__buffer_unlock_commit(buffer, event);

return 1;

}

EXPORT_SYMBOL_GPL(__trace_bputs);

#ifdef CONFIG_TRACER_SNAPSHOT

/**

* trace_snapshot - take a snapshot of the current buffer.

*

* This causes a swap between the snapshot buffer and the current live

* tracing buffer. You can use this to take snapshots of the live

* trace when some condition is triggered, but continue to trace.

*

* Note, make sure to allocate the snapshot with either

* a tracing_snapshot_alloc(), or by doing it manually

* with: echo 1 > /sys/kernel/debug/tracing/snapshot

*

* If the snapshot buffer is not allocated, it will stop tracing.

* Basically making a permanent snapshot.

*/

void tracing_snapshot(void)

{

struct trace_array *tr = &global_trace;

struct tracer *tracer = tr->current_trace;

unsigned long flags;

if (in_nmi()) {

internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");

internal_trace_puts("*** snapshot is being ignored ***\n");

return;

}

if (!tr->allocated_snapshot) {

internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");

internal_trace_puts("*** stopping trace here! ***\n");

tracing_off();

return;

}

/* Note, snapshot can not be used when the tracer uses it */

if (tracer->use_max_tr) {

internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");

internal_trace_puts("*** Can not use snapshot (sorry) ***\n");

return;

}

local_irq_save(flags);

update_max_tr(tr, current, smp_processor_id());

local_irq_restore(flags);

}

EXPORT_SYMBOL_GPL(tracing_snapshot);

static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,

struct trace_buffer *size_buf, int cpu_id);

static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);

static int alloc_snapshot(struct trace_array *tr)

{

int ret;

if (!tr->allocated_snapshot) {

/* allocate spare buffer */

ret = resize_buffer_duplicate_size(&tr->max_buffer,

&tr->trace_buffer, RING_BUFFER_ALL_CPUS);

if (ret < 0)

return ret;

tr->allocated_snapshot = true;

}

return 0;

}

void free_snapshot(struct trace_array *tr)

{

/*

* We don't free the ring buffer. instead, resize it because

* The max_tr ring buffer has some state (e.g. ring->clock) and

* we want preserve it.

*/

ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);

set_buffer_entries(&tr->max_buffer, 1);

tracing_reset_online_cpus(&tr->max_buffer);

tr->allocated_snapshot = false;

}

/**

* trace_snapshot_alloc - allocate and take a snapshot of the current buffer.

*

* This is similar to trace_snapshot(), but it will allocate the

* snapshot buffer if it isn't already allocated. Use this only

* where it is safe to sleep, as the allocation may sleep.

*

* This causes a swap between the snapshot buffer and the current live

* tracing buffer. You can use this to take snapshots of the live

* trace when some condition is triggered, but continue to trace.

*/

void tracing_snapshot_alloc(void)

{

struct trace_array *tr = &global_trace;

int ret;

ret = alloc_snapshot(tr);

if (WARN_ON(ret < 0))

return;

tracing_snapshot();

}

EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);

#else

void tracing_snapshot(void)

{

WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");

}

EXPORT_SYMBOL_GPL(tracing_snapshot);

void tracing_snapshot_alloc(void)

{

/* Give warning */

tracing_snapshot();

}

EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);

#endif /* CONFIG_TRACER_SNAPSHOT */

static void tracer_tracing_off(struct trace_array *tr)

{

if (tr->trace_buffer.buffer)

ring_buffer_record_off(tr->trace_buffer.buffer);

/*

* This flag is looked at when buffers haven't been allocated

* yet, or by some tracers (like irqsoff), that just want to

* know if the ring buffer has been disabled, but it can handle

* races of where it gets disabled but we still do a record.

* As the check is in the fast path of the tracers, it is more

* important to be fast than accurate.

*/

tr->buffer_disabled = 1;

/* Make the flag seen by readers */

smp_wmb();

}

/**

* tracing_off - turn off tracing buffers

*

* This function stops the tracing buffers from recording data.

* It does not disable any overhead the tracers themselves may

* be causing. This function simply causes all recording to

* the ring buffers to fail.

*/

void tracing_off(void)

{

tracer_tracing_off(&global_trace);

}

EXPORT_SYMBOL_GPL(tracing_off);

void disable_trace_on_warning(void)

{

if (__disable_trace_on_warning)

tracing_off();

}

/**

* tracer_tracing_is_on - show real state of ring buffer enabled

* @tr : the trace array to know if ring buffer is enabled

*

* Shows real state of the ring buffer if it is enabled or not.

*/

static int tracer_tracing_is_on(struct trace_array *tr)

{

if (tr->trace_buffer.buffer)

return ring_buffer_record_is_on(tr->trace_buffer.buffer);

return !tr->buffer_disabled;

}

/**

* tracing_is_on - show state of ring buffers enabled

*/

int tracing_is_on(void)

{

return tracer_tracing_is_on(&global_trace);

}

EXPORT_SYMBOL_GPL(tracing_is_on);

static int __init set_buf_size(char *str)

{

unsigned long buf_size;

if (!str)

return 0;

buf_size = memparse(str, &str);

/* nr_entries can not be zero */

if (buf_size == 0)

return 0;

trace_buf_size = buf_size;

return 1;

}

__setup("trace_buf_size=", set_buf_size);

static int __init set_tracing_thresh(char *str)

{

unsigned long threshold;

int ret;

if (!str)

return 0;

ret = kstrtoul(str, 0, &threshold);

if (ret < 0)

return 0;

tracing_thresh = threshold * 1000;

return 1;

}

__setup("tracing_thresh=", set_tracing_thresh);

unsigned long nsecs_to_usecs(unsigned long nsecs)

{

return nsecs / 1000;

}

/* These must match the bit postions in trace_iterator_flags */

static const char *trace_options[] = {

"print-parent",

"sym-offset",

"sym-addr",

"verbose",

"raw",

"hex",

"bin",

"block",

"stacktrace",

"trace_printk",

"ftrace_preempt",

"branch",

"annotate",

"userstacktrace",

"sym-userobj",

"printk-msg-only",

"context-info",

"latency-format",

"sleep-time",

"graph-time",

"record-cmd",

"overwrite",

"disable_on_free",

"irq-info",

"markers",

"function-trace",

NULL

};

static struct {

u64 (*func)(void);

const char *name;

int in_ns; /* is this clock in nanoseconds? */

} trace_clocks[] = {

{ trace_clock_local, "local", 1 },

{ trace_clock_global, "global", 1 },

{ trace_clock_counter, "counter", 0 },

{ trace_clock_jiffies, "uptime", 1 },

{ trace_clock, "perf", 1 },

ARCH_TRACE_CLOCKS

};

/*

* trace_parser_get_init - gets the buffer for trace parser

*/

int trace_parser_get_init(struct trace_parser *parser, int size)

{

memset(parser, 0, sizeof(*parser));

parser->buffer = kmalloc(size, GFP_KERNEL);

if (!parser->buffer)

return 1;

parser->size = size;

return 0;

}

/*

* trace_parser_put - frees the buffer for trace parser

*/

void trace_parser_put(struct trace_parser *parser)

{

kfree(parser->buffer);

}

/*

* trace_get_user - reads the user input string separated by space

* (matched by isspace(ch))

*

* For each string found the 'struct trace_parser' is updated,

* and the function returns.

*

* Returns number of bytes read.

*

* See kernel/trace/trace.h for 'struct trace_parser' details.

*/

int trace_get_user(struct trace_parser *parser, const char __user *ubuf,

size_t cnt, loff_t *ppos)

{

char ch;

size_t read = 0;

ssize_t ret;

if (!*ppos)

trace_parser_clear(parser);

ret = get_user(ch, ubuf++);

if (ret)

goto out;

read++;

cnt--;

/*

* The parser is not finished with the last write,

* continue reading the user input without skipping spaces.

*/

if (!parser->cont) {

/* skip white space */

while (cnt && isspace(ch)) {

ret = get_user(ch, ubuf++);

if (ret)

goto out;

read++;

cnt--;

}

/* only spaces were written */

if (isspace(ch)) {

*ppos += read;

ret = read;

goto out;

}

parser->idx = 0;

}

/* read the non-space input */

while (cnt && !isspace(ch)) {

if (parser->idx < parser->size - 1)

parser->buffer[parser->idx++] = ch;

else {

ret = -EINVAL;

goto out;

}

ret = get_user(ch, ubuf++);

if (ret)

goto out;

read++;

cnt--;

}

/* We either got finished input or we have to wait for another call. */

if (isspace(ch)) {

parser->buffer[parser->idx] = 0;

parser->cont = false;

} else if (parser->idx < parser->size - 1) {

parser->cont = true;

parser->buffer[parser->idx++] = ch;

} else {

ret = -EINVAL;

goto out;

}

*ppos += read;

ret = read;

out:

return ret;

}

ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)

{

int len;

int ret;

if (!cnt)

return 0;

if (s->len <= s->readpos)

return -EBUSY;

len = s->len - s->readpos;

if (cnt > len)

cnt = len;

ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);

if (ret == cnt)

return -EFAULT;

cnt -= ret;

s->readpos += cnt;

return cnt;

}

static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)

{

int len;

if (s->len <= s->readpos)

return -EBUSY;

len = s->len - s->readpos;

if (cnt > len)

cnt = len;

memcpy(buf, s->buffer + s->readpos, cnt);

s->readpos += cnt;

return cnt;

}

/*

* ftrace_max_lock is used to protect the swapping of buffers

* when taking a max snapshot. The buffers themselves are

* protected by per_cpu spinlocks. But the action of the swap

* needs its own lock.

*

* This is defined as a arch_spinlock_t in order to help

* with performance when lockdep debugging is enabled.

*

* It is also used in other places outside the update_max_tr

* so it needs to be defined outside of the

* CONFIG_TRACER_MAX_TRACE.

*/

static arch_spinlock_t ftrace_max_lock =

(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;

unsigned long __read_mostly tracing_thresh;

#ifdef CONFIG_TRACER_MAX_TRACE

unsigned long __read_mostly tracing_max_latency;

/*

* Copy the new maximum trace into the separate maximum-trace

* structure. (this way the maximum trace is permanently saved,

* for later retrieval via /sys/kernel/debug/tracing/latency_trace)

*/

static void

__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)

{

struct trace_buffer *trace_buf = &tr->trace_buffer;

struct trace_buffer *max_buf = &tr->max_buffer;

struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);

struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);

max_buf->cpu = cpu;

max_buf->time_start = data->preempt_timestamp;

max_data->saved_latency = tracing_max_latency;

max_data->critical_start = data->critical_start;

max_data->critical_end = data->critical_end;

memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);

max_data->pid = tsk->pid;

/*

* If tsk == current, then use current_uid(), as that does not use

* RCU. The irq tracer can be called out of RCU scope.

*/

if (tsk == current)

max_data->uid = current_uid();

else

max_data->uid = task_uid(tsk);

max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;

max_data->policy = tsk->policy;

max_data->rt_priority = tsk->rt_priority;

/* record this tasks comm */

tracing_record_cmdline(tsk);

}

/**

* update_max_tr - snapshot all trace buffers from global_trace to max_tr

* @tr: tracer

* @tsk: the task with the latency

* @cpu: The cpu that initiated the trace.

*

* Flip the buffers between the @tr and the max_tr and record information

* about which task was the cause of this latency.

*/

void

update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)

{

struct ring_buffer *buf;

if (tr->stop_count)

return;

WARN_ON_ONCE(!irqs_disabled());

if (!tr->allocated_snapshot) {