#include <linux/ring_buffer_types.h>

#include <linux/sched/isolation.h>

#include <linux/trace_recursion.h>

#include <linux/panic_notifier.h>

#include <linux/trace_events.h>

#include <linux/ring_buffer.h>

#include <linux/trace_clock.h>

#include <linux/sched/clock.h>

#include <linux/cacheflush.h>

#include <linux/trace_seq.h>

#include <linux/spinlock.h>

#include <linux/irq_work.h>

#include <linux/security.h>

#include <linux/uaccess.h>

#include <linux/hardirq.h>

#include <linux/kthread.h> /* for self test */

#include <linux/module.h>

#include <linux/percpu.h>

#include <linux/mutex.h>

#include <linux/delay.h>

#include <linux/slab.h>

#include <linux/init.h>

#include <linux/hash.h>

#include <linux/list.h>

#include <linux/cpu.h>

#include <linux/oom.h>

#include <linux/mm.h>

#include <asm/ring_buffer.h>

#include <asm/local64.h>

#include <asm/local.h>

#include <asm/setup.h>

#include "trace.h"

#define TS_MSB (0xf8ULL << 56)

#define ABS_TS_MASK (~TS_MSB)

static void update_pages_handler(struct work_struct *work);

#define RING_BUFFER_META_MAGIC 0xBADFEED

struct ring_buffer_meta {

int magic;

int struct_sizes;

unsigned long total_size;

unsigned long buffers_offset;

};

struct ring_buffer_cpu_meta {

unsigned long first_buffer;

unsigned long head_buffer;

unsigned long commit_buffer;

__u32 subbuf_size;

__u32 nr_subbufs;

#ifdef CONFIG_RING_BUFFER_PERSISTENT_INJECT

__u32 nr_invalid;

__u32 entry_bytes;

int buffers[];

};

int ring_buffer_print_entry_header(struct trace_seq *s)

{

trace_seq_puts(s, "# compressed entry header\n");

trace_seq_puts(s, "\ttype_len : 5 bits\n");

trace_seq_puts(s, "\ttime_delta : 27 bits\n");

trace_seq_puts(s, "\tarray : 32 bits\n");

trace_seq_putc(s, '\n');

trace_seq_printf(s, "\tpadding : type == %d\n",

RINGBUF_TYPE_PADDING);

trace_seq_printf(s, "\ttime_extend : type == %d\n",

RINGBUF_TYPE_TIME_EXTEND);

trace_seq_printf(s, "\ttime_stamp : type == %d\n",

RINGBUF_TYPE_TIME_STAMP);

trace_seq_printf(s, "\tdata max type_len == %d\n",

RINGBUF_TYPE_DATA_TYPE_LEN_MAX);

return !trace_seq_has_overflowed(s);

}

#define RB_BUFFER_OFF (1 << 20)

#define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX

enum {

RB_LEN_TIME_EXTEND = 8,

RB_LEN_TIME_STAMP = 8,

};

#define skip_time_extend(event) \

((struct ring_buffer_event *)((char *)event + RB_LEN_TIME_EXTEND))

#define extended_time(event) \

(event->type_len >= RINGBUF_TYPE_TIME_EXTEND)

static inline bool rb_null_event(struct ring_buffer_event *event)

{

return event->type_len == RINGBUF_TYPE_PADDING && !event->time_delta;

}

static void rb_event_set_padding(struct ring_buffer_event *event)

{

/* padding has a NULL time_delta */

event->type_len = RINGBUF_TYPE_PADDING;

event->time_delta = 0;

}

static unsigned

rb_event_data_length(struct ring_buffer_event *event)

{

unsigned length;

if (event->type_len)

length = event->type_len * RB_ALIGNMENT;

else

length = event->array[0];

return length + RB_EVNT_HDR_SIZE;

}

static inline unsigned

rb_event_length(struct ring_buffer_event *event)

{

switch (event->type_len) {

case RINGBUF_TYPE_PADDING:

if (rb_null_event(event))

/* undefined */

return -1;

return event->array[0] + RB_EVNT_HDR_SIZE;

case RINGBUF_TYPE_TIME_EXTEND:

return RB_LEN_TIME_EXTEND;

case RINGBUF_TYPE_TIME_STAMP:

return RB_LEN_TIME_STAMP;

case RINGBUF_TYPE_DATA:

return rb_event_data_length(event);

default:

WARN_ON_ONCE(1);

}

/* not hit */

return 0;

}

static inline unsigned

rb_event_ts_length(struct ring_buffer_event *event)

{

unsigned len = 0;

if (extended_time(event)) {

/* time extends include the data event after it */

len = RB_LEN_TIME_EXTEND;

event = skip_time_extend(event);

}

return len + rb_event_length(event);

}

unsigned ring_buffer_event_length(struct ring_buffer_event *event)

{

unsigned length;

if (extended_time(event))

event = skip_time_extend(event);

length = rb_event_length(event);

if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX)

return length;

length -= RB_EVNT_HDR_SIZE;

if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]) ||

RB_FORCE_8BYTE_ALIGNMENT)

length -= sizeof(event->array[0]);

return length;

}

EXPORT_SYMBOL_GPL(ring_buffer_event_length);

static __always_inline void *

rb_event_data(struct ring_buffer_event *event)

{

if (extended_time(event))

event = skip_time_extend(event);

WARN_ON_ONCE(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX);

/* If length is in len field, then array[0] has the data */

if (event->type_len)

return (void *)&event->array[0];

/* Otherwise length is in array[0] and array[1] has the data */

return (void *)&event->array[1];

}

void *ring_buffer_event_data(struct ring_buffer_event *event)

{

return rb_event_data(event);

}

EXPORT_SYMBOL_GPL(ring_buffer_event_data);

#define for_each_buffer_cpu(buffer, cpu) \

for_each_cpu(cpu, buffer->cpumask)

#define for_each_online_buffer_cpu(buffer, cpu) \

for_each_cpu_and(cpu, buffer->cpumask, cpu_online_mask)

static u64 rb_event_time_stamp(struct ring_buffer_event *event)

{

u64 ts;

ts = event->array[0];

ts <<= TS_SHIFT;

ts += event->time_delta;

return ts;

}

#define RB_MISSED_EVENTS (1 << 31)

#define RB_MISSED_STORED (1 << 30)

#define RB_MISSED_MASK (3 << 30)

struct buffer_data_read_page {

unsigned order; /* order of the page */

struct buffer_data_page *data; /* actual data, stored in this page */

};

struct buffer_page {

struct list_head list; /* list of buffer pages */

local_t write; /* index for next write */

unsigned read; /* index for next read */

local_t entries; /* entries on this page */

unsigned long real_end; /* real end of data */

unsigned order; /* order of the page */

u32 id:30; /* ID for external mapping */

u32 range:1; /* Mapped via a range */

struct buffer_data_page *page; /* Actual data page */

};

#define RB_WRITE_MASK 0xfffff

#define RB_WRITE_INTCNT (1 << 20)

static void rb_init_data_page(struct buffer_data_page *bpage)

{

local_set(&bpage->commit, 0);

bpage->time_stamp = 0;

}

static __always_inline long rb_data_page_commit(struct buffer_data_page *dpage)

{

return local_read(&dpage->commit);

}

static __always_inline long rb_data_page_size(struct buffer_data_page *dpage)

{

return rb_data_page_commit(dpage) & ~RB_MISSED_MASK;

}

static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage)

{

return rb_data_page_commit(bpage->page);

}

static __always_inline unsigned int rb_page_size(struct buffer_page *bpage)

{

return rb_data_page_size(bpage->page);

}

static void free_buffer_page(struct buffer_page *bpage)

{

/* Range pages are not to be freed */

if (!bpage->range)

free_pages((unsigned long)bpage->page, bpage->order);

kfree(bpage);

}

#define alloc_cpu_buffer(cpu) (struct ring_buffer_per_cpu *) \

kzalloc_node(ALIGN(sizeof(struct ring_buffer_per_cpu), \

cache_line_size()), GFP_KERNEL, cpu_to_node(cpu));

#define alloc_cpu_page(cpu) (struct buffer_page *) \

kzalloc_node(ALIGN(sizeof(struct buffer_page), \

cache_line_size()), GFP_KERNEL, cpu_to_node(cpu));

static struct buffer_data_page *alloc_cpu_data(int cpu, int order)

{

struct buffer_data_page *dpage;

struct page *page;

gfp_t mflags;

/*

mflags = GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_COMP | __GFP_ZERO;

page = alloc_pages_node(cpu_to_node(cpu), mflags, order);

if (!page)

return NULL;

dpage = page_address(page);

rb_init_data_page(dpage);

return dpage;

}

struct rb_irq_work {

struct irq_work work;

wait_queue_head_t waiters;

wait_queue_head_t full_waiters;

atomic_t seq;

bool waiters_pending;

bool full_waiters_pending;

bool wakeup_full;

};

struct rb_event_info {

u64 ts;

u64 delta;

u64 before;

u64 after;

unsigned long length;

struct buffer_page *tail_page;

int add_timestamp;

};

enum {

RB_ADD_STAMP_NONE = 0,

RB_ADD_STAMP_EXTEND = BIT(1),

RB_ADD_STAMP_ABSOLUTE = BIT(2),

RB_ADD_STAMP_FORCE = BIT(3)

};

enum {

RB_CTX_TRANSITION,

RB_CTX_NMI,

RB_CTX_IRQ,

RB_CTX_SOFTIRQ,

RB_CTX_NORMAL,

RB_CTX_MAX

};

struct rb_time_struct {

local64_t time;

};

typedef struct rb_time_struct rb_time_t;

#define MAX_NEST 5

struct ring_buffer_per_cpu {

int cpu;

atomic_t record_disabled;

atomic_t resize_disabled;

struct trace_buffer *buffer;

raw_spinlock_t reader_lock; /* serialize readers */

arch_spinlock_t lock;

struct lock_class_key lock_key;

struct buffer_data_page *free_page;

unsigned long nr_pages;

unsigned int current_context;

struct list_head *pages;

/* pages generation counter, incremented when the list changes */

unsigned long cnt;

struct buffer_page *head_page; /* read from head */

struct buffer_page *tail_page; /* write to tail */

struct buffer_page *commit_page; /* committed pages */

struct buffer_page *reader_page;

unsigned long lost_events;

unsigned long last_overrun;

unsigned long nest;

local_t entries_bytes;

local_t entries;

local_t overrun;

local_t commit_overrun;

local_t dropped_events;

local_t committing;

local_t commits;

local_t pages_touched;

local_t pages_lost;

local_t pages_read;

long last_pages_touch;

size_t shortest_full;

unsigned long read;

unsigned long read_bytes;

rb_time_t write_stamp;

rb_time_t before_stamp;

u64 event_stamp[MAX_NEST];

u64 read_stamp;

/* pages removed since last reset */

unsigned long pages_removed;

unsigned int mapped;

unsigned int user_mapped; /* user space mapping */

struct mutex mapping_lock;

struct buffer_page **subbuf_ids; /* ID to subbuf VA */

struct trace_buffer_meta *meta_page;

struct ring_buffer_cpu_meta *ring_meta;

struct ring_buffer_remote *remote;

/* ring buffer pages to update, > 0 to add, < 0 to remove */

long nr_pages_to_update;

struct list_head new_pages; /* new pages to add */

struct work_struct update_pages_work;

struct completion update_done;

struct rb_irq_work irq_work;

};

struct trace_buffer {

unsigned flags;

int cpus;

atomic_t record_disabled;

atomic_t resizing;

cpumask_var_t cpumask;

struct lock_class_key *reader_lock_key;

struct mutex mutex;

struct ring_buffer_per_cpu **buffers;

struct ring_buffer_remote *remote;

struct hlist_node node;

u64 (*clock)(void);

struct rb_irq_work irq_work;

bool time_stamp_abs;

unsigned long range_addr_start;

unsigned long range_addr_end;

struct notifier_block flush_nb;

struct ring_buffer_meta *meta;

unsigned int subbuf_size;

unsigned int subbuf_order;

unsigned int max_data_size;

};

struct ring_buffer_iter {

struct ring_buffer_per_cpu *cpu_buffer;

unsigned long head;

unsigned long next_event;

struct buffer_page *head_page;

struct buffer_page *cache_reader_page;

unsigned long cache_read;

unsigned long cache_pages_removed;

u64 read_stamp;

u64 page_stamp;

struct ring_buffer_event *event;

size_t event_size;

int missed_events;

};

int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq *s)

{

struct buffer_data_page field;

trace_seq_printf(s, "\tfield: u64 timestamp;\t"

"offset:0;\tsize:%u;\tsigned:%u;\n",

(unsigned int)sizeof(field.time_stamp),

(unsigned int)is_signed_type(u64));

trace_seq_printf(s, "\tfield: local_t commit;\t"

"offset:%u;\tsize:%u;\tsigned:%u;\n",

(unsigned int)offsetof(typeof(field), commit),

(unsigned int)sizeof(field.commit),

(unsigned int)is_signed_type(long));

trace_seq_printf(s, "\tfield: char overwrite;\t"

"offset:%u;\tsize:%u;\tsigned:%u;\n",

(unsigned int)offsetof(typeof(field), commit),

1,

(unsigned int)is_signed_type(char));

trace_seq_printf(s, "\tfield: char data;\t"

"offset:%u;\tsize:%u;\tsigned:%u;\n",

(unsigned int)offsetof(typeof(field), data),

(unsigned int)(buffer ? buffer->subbuf_size :

PAGE_SIZE - BUF_PAGE_HDR_SIZE),

(unsigned int)is_signed_type(char));

return !trace_seq_has_overflowed(s);

}

static inline void rb_time_read(rb_time_t *t, u64 *ret)

{

*ret = local64_read(&t->time);

}

static void rb_time_set(rb_time_t *t, u64 val)

{

local64_set(&t->time, val);

}

#ifdef RB_VERIFY_EVENT

static struct list_head *rb_list_head(struct list_head *list);

static void verify_event(struct ring_buffer_per_cpu *cpu_buffer,

void *event)

{

struct buffer_page *page = cpu_buffer->commit_page;

struct buffer_page *tail_page = READ_ONCE(cpu_buffer->tail_page);

struct list_head *next;

long commit, write;

unsigned long addr = (unsigned long)event;

bool done = false;

int stop = 0;

/* Make sure the event exists and is not committed yet */

do {

if (page == tail_page || WARN_ON_ONCE(stop++ > 100))

done = true;

commit = rb_page_commit(page);

write = local_read(&page->write);

if (addr >= (unsigned long)&page->page->data[commit] &&

addr < (unsigned long)&page->page->data[write])

return;

next = rb_list_head(page->list.next);

page = list_entry(next, struct buffer_page, list);

} while (!done);

WARN_ON_ONCE(1);

}

static inline void verify_event(struct ring_buffer_per_cpu *cpu_buffer,

void *event)

{

}

static inline u64 rb_fix_abs_ts(u64 abs, u64 save_ts)

{

if (save_ts & TS_MSB) {

abs |= save_ts & TS_MSB;

/* Check for overflow */

if (unlikely(abs < save_ts))

abs += 1ULL << 59;

}

return abs;

}

static inline u64 rb_time_stamp(struct trace_buffer *buffer);

u64 ring_buffer_event_time_stamp(struct trace_buffer *buffer,

struct ring_buffer_event *event)

{

struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[smp_processor_id()];

unsigned int nest;

u64 ts;

/* If the event includes an absolute time, then just use that */

if (event->type_len == RINGBUF_TYPE_TIME_STAMP) {

ts = rb_event_time_stamp(event);

return rb_fix_abs_ts(ts, cpu_buffer->tail_page->page->time_stamp);

}

nest = local_read(&cpu_buffer->committing);

verify_event(cpu_buffer, event);

if (WARN_ON_ONCE(!nest))

goto fail;

/* Read the current saved nesting level time stamp */

if (likely(--nest < MAX_NEST))

return cpu_buffer->event_stamp[nest];

/* Shouldn't happen, warn if it does */

WARN_ONCE(1, "nest (%d) greater than max", nest);

fail:

rb_time_read(&cpu_buffer->write_stamp, &ts);

return ts;

}

size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu)

{

size_t read;

size_t lost;

size_t cnt;

read = local_read(&buffer->buffers[cpu]->pages_read);

lost = local_read(&buffer->buffers[cpu]->pages_lost);

cnt = local_read(&buffer->buffers[cpu]->pages_touched);

if (WARN_ON_ONCE(cnt < lost))

return 0;

cnt -= lost;

/* The reader can read an empty page, but not more than that */

if (cnt < read) {

WARN_ON_ONCE(read > cnt + 1);

return 0;

}

return cnt - read;

}

static __always_inline bool full_hit(struct trace_buffer *buffer, int cpu, int full)

{

struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];

size_t nr_pages;

size_t dirty;

nr_pages = cpu_buffer->nr_pages;

if (!nr_pages || !full)

return true;

/*

dirty = ring_buffer_nr_dirty_pages(buffer, cpu) + 1;

return (dirty * 100) >= (full * nr_pages);

}

static void rb_wake_up_waiters(struct irq_work *work)

{

struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work);

/* For waiters waiting for the first wake up */

(void)atomic_fetch_inc_release(&rbwork->seq);

wake_up_all(&rbwork->waiters);

if (rbwork->full_waiters_pending || rbwork->wakeup_full) {

/* Only cpu_buffer sets the above flags */

struct ring_buffer_per_cpu *cpu_buffer =

container_of(rbwork, struct ring_buffer_per_cpu, irq_work);

/* Called from interrupt context */

raw_spin_lock(&cpu_buffer->reader_lock);

rbwork->wakeup_full = false;

rbwork->full_waiters_pending = false;

/* Waking up all waiters, they will reset the shortest full */

cpu_buffer->shortest_full = 0;

raw_spin_unlock(&cpu_buffer->reader_lock);

wake_up_all(&rbwork->full_waiters);

}

}

void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu)

{

struct ring_buffer_per_cpu *cpu_buffer;

struct rb_irq_work *rbwork;

if (!buffer)

return;

if (cpu == RING_BUFFER_ALL_CPUS) {

/* Wake up individual ones too. One level recursion */

for_each_buffer_cpu(buffer, cpu)

ring_buffer_wake_waiters(buffer, cpu);

rbwork = &buffer->irq_work;

} else {

if (WARN_ON_ONCE(!buffer->buffers))

return;

if (WARN_ON_ONCE(cpu >= nr_cpu_ids))

return;

cpu_buffer = buffer->buffers[cpu];

/* The CPU buffer may not have been initialized yet */

if (!cpu_buffer)

return;

rbwork = &cpu_buffer->irq_work;

}

/* This can be called in any context */

irq_work_queue(&rbwork->work);

}

static bool rb_watermark_hit(struct trace_buffer *buffer, int cpu, int full)

{

struct ring_buffer_per_cpu *cpu_buffer;

bool ret = false;

/* Reads of all CPUs always waits for any data */

if (cpu == RING_BUFFER_ALL_CPUS)

return !ring_buffer_empty(buffer);

cpu_buffer = buffer->buffers[cpu];

if (!ring_buffer_empty_cpu(buffer, cpu)) {

unsigned long flags;

bool pagebusy;

if (!full)

return true;

raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;

ret = !pagebusy && full_hit(buffer, cpu, full);

if (!ret && (!cpu_buffer->shortest_full ||

cpu_buffer->shortest_full > full)) {

cpu_buffer->shortest_full = full;

}

raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

}

return ret;

}

static inline bool

rb_wait_cond(struct rb_irq_work *rbwork, struct trace_buffer *buffer,

int cpu, int full, ring_buffer_cond_fn cond, void *data)

{

if (rb_watermark_hit(buffer, cpu, full))

return true;

if (cond(data))

return true;

/*

if (full)

rbwork->full_waiters_pending = true;

else

rbwork->waiters_pending = true;

return false;

}

struct rb_wait_data {

struct rb_irq_work *irq_work;

int seq;

};

static bool rb_wait_once(void *data)

{

struct rb_wait_data *rdata = data;

struct rb_irq_work *rbwork = rdata->irq_work;

return atomic_read_acquire(&rbwork->seq) != rdata->seq;

}

int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full,

ring_buffer_cond_fn cond, void *data)

{

struct ring_buffer_per_cpu *cpu_buffer;

struct wait_queue_head *waitq;

struct rb_irq_work *rbwork;

struct rb_wait_data rdata;

int ret = 0;

/*

if (cpu == RING_BUFFER_ALL_CPUS) {

rbwork = &buffer->irq_work;

/* Full only makes sense on per cpu reads */

full = 0;

} else {

if (!cpumask_test_cpu(cpu, buffer->cpumask))

return -ENODEV;