// SPDX-License-Identifier: GPL-2.0

/* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com

* Copyright (c) 2016 Facebook

*/

#include <linux/kernel.h>

#include <linux/types.h>

#include <linux/slab.h>

#include <linux/bpf.h>

#include <linux/bpf_verifier.h>

#include <linux/bpf_perf_event.h>

#include <linux/btf.h>

#include <linux/filter.h>

#include <linux/uaccess.h>

#include <linux/ctype.h>

#include <linux/kprobes.h>

#include <linux/spinlock.h>

#include <linux/syscalls.h>

#include <linux/error-injection.h>

#include <linux/btf_ids.h>

#include <linux/bpf_lsm.h>

#include <linux/fprobe.h>

#include <linux/bsearch.h>

#include <linux/sort.h>

#include <linux/key.h>

#include <linux/namei.h>

#include <linux/file.h>

#include <net/bpf_sk_storage.h>

#include <uapi/linux/bpf.h>

#include <uapi/linux/btf.h>

#include <asm/tlb.h>

#include "trace_probe.h"

#include "trace.h"

#define CREATE_TRACE_POINTS

#include "bpf_trace.h"

#define bpf_event_rcu_dereference(p) \

rcu_dereference_protected(p, lockdep_is_held(&bpf_event_mutex))

#define MAX_UPROBE_MULTI_CNT (1U << 20)

#define MAX_KPROBE_MULTI_CNT (1U << 20)

#define MAX_TRACING_MULTI_CNT (1U << 20)

#ifdef CONFIG_MODULES

struct bpf_trace_module {

struct module *module;

struct list_head list;

};

static LIST_HEAD(bpf_trace_modules);

static DEFINE_MUTEX(bpf_module_mutex);

static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name)

{

struct bpf_raw_event_map *btp, *ret = NULL;

struct bpf_trace_module *btm;

unsigned int i;

mutex_lock(&bpf_module_mutex);

list_for_each_entry(btm, &bpf_trace_modules, list) {

for (i = 0; i < btm->module->num_bpf_raw_events; ++i) {

btp = &btm->module->bpf_raw_events[i];

if (!strcmp(btp->tp->name, name)) {

if (try_module_get(btm->module))

ret = btp;

goto out;

}

}

}

out:

mutex_unlock(&bpf_module_mutex);

return ret;

}

#else

static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name)

{

return NULL;

}

#endif /* CONFIG_MODULES */

u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);

u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);

static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size,

u64 flags, const struct btf **btf,

s32 *btf_id);

static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx);

static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx);

static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx);

static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx);

/**

* trace_call_bpf - invoke BPF program

* @call: tracepoint event

* @ctx: opaque context pointer

*

* kprobe handlers execute BPF programs via this helper.

* Can be used from static tracepoints in the future.

*

* Return: BPF programs always return an integer which is interpreted by

* kprobe handler as:

* 0 - return from kprobe (event is filtered out)

* 1 - store kprobe event into ring buffer

* Other values are reserved and currently alias to 1

*/

unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)

{

unsigned int ret;

cant_sleep();

if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {

/*

* since some bpf program is already running on this cpu,

* don't call into another bpf program (same or different)

* and don't send kprobe event into ring-buffer,

* so return zero here

*/

rcu_read_lock();

bpf_prog_inc_misses_counters(rcu_dereference(call->prog_array));

rcu_read_unlock();

ret = 0;

goto out;

}

/*

* Instead of moving rcu_read_lock/rcu_dereference/rcu_read_unlock

* to all call sites, we did a bpf_prog_array_valid() there to check

* whether call->prog_array is empty or not, which is

* a heuristic to speed up execution.

*

* If bpf_prog_array_valid() fetched prog_array was

* non-NULL, we go into trace_call_bpf() and do the actual

* proper rcu_dereference() under RCU lock.

* If it turns out that prog_array is NULL then, we bail out.

* For the opposite, if the bpf_prog_array_valid() fetched pointer

* was NULL, you'll skip the prog_array with the risk of missing

* out of events when it was updated in between this and the

* rcu_dereference() which is accepted risk.

*/

rcu_read_lock();

ret = bpf_prog_run_array(rcu_dereference(call->prog_array),

ctx, bpf_prog_run);

rcu_read_unlock();

out:

__this_cpu_dec(bpf_prog_active);

return ret;

}

/**

* trace_call_bpf_faultable - invoke BPF program in faultable context

* @call: tracepoint event

* @ctx: opaque context pointer

*

* Variant of trace_call_bpf() for faultable tracepoints (syscall

* tracepoints). Supports sleepable BPF programs by using rcu_tasks_trace

* for lifetime protection and bpf_prog_run_array_sleepable() for per-program

* RCU flavor selection, following the uprobe pattern.

*

* Per-program recursion protection is provided by

* bpf_prog_run_array_sleepable(). Global bpf_prog_active is not

* needed because syscall tracepoints cannot self-recurse.

*

* Must be called from a faultable/preemptible context.

*/

unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx)

{

struct bpf_prog_array *prog_array;

might_fault();

guard(rcu_tasks_trace)();

prog_array = rcu_dereference_check(call->prog_array,

rcu_read_lock_trace_held());

return bpf_prog_run_array_sleepable(prog_array, ctx, bpf_prog_run);

}

#ifdef CONFIG_BPF_KPROBE_OVERRIDE

BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc)

{

regs_set_return_value(regs, rc);

override_function_with_return(regs);

return 0;

}

static const struct bpf_func_proto bpf_override_return_proto = {

.func = bpf_override_return,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_CTX,

.arg2_type = ARG_ANYTHING,

};

#endif

static __always_inline int

bpf_probe_read_user_common(void *dst, u32 size, const void __user *unsafe_ptr)

{

int ret;

ret = copy_from_user_nofault(dst, unsafe_ptr, size);

if (unlikely(ret < 0))

memset(dst, 0, size);

return ret;

}

BPF_CALL_3(bpf_probe_read_user, void *, dst, u32, size,

const void __user *, unsafe_ptr)

{

return bpf_probe_read_user_common(dst, size, unsafe_ptr);

}

const struct bpf_func_proto bpf_probe_read_user_proto = {

.func = bpf_probe_read_user,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_UNINIT_MEM,

.arg2_type = ARG_CONST_SIZE_OR_ZERO,

.arg3_type = ARG_ANYTHING,

};

static __always_inline int

bpf_probe_read_user_str_common(void *dst, u32 size,

const void __user *unsafe_ptr)

{

int ret;

/*

* NB: We rely on strncpy_from_user() not copying junk past the NUL

* terminator into `dst`.

*

* strncpy_from_user() does long-sized strides in the fast path. If the

* strncpy does not mask out the bytes after the NUL in `unsafe_ptr`,

* then there could be junk after the NUL in `dst`. If user takes `dst`

* and keys a hash map with it, then semantically identical strings can

* occupy multiple entries in the map.

*/

ret = strncpy_from_user_nofault(dst, unsafe_ptr, size);

if (unlikely(ret < 0))

memset(dst, 0, size);

return ret;

}

BPF_CALL_3(bpf_probe_read_user_str, void *, dst, u32, size,

const void __user *, unsafe_ptr)

{

return bpf_probe_read_user_str_common(dst, size, unsafe_ptr);

}

const struct bpf_func_proto bpf_probe_read_user_str_proto = {

.func = bpf_probe_read_user_str,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_UNINIT_MEM,

.arg2_type = ARG_CONST_SIZE_OR_ZERO,

.arg3_type = ARG_ANYTHING,

};

BPF_CALL_3(bpf_probe_read_kernel, void *, dst, u32, size,

const void *, unsafe_ptr)

{

return bpf_probe_read_kernel_common(dst, size, unsafe_ptr);

}

const struct bpf_func_proto bpf_probe_read_kernel_proto = {

.func = bpf_probe_read_kernel,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_UNINIT_MEM,

.arg2_type = ARG_CONST_SIZE_OR_ZERO,

.arg3_type = ARG_ANYTHING,

};

static __always_inline int

bpf_probe_read_kernel_str_common(void *dst, u32 size, const void *unsafe_ptr)

{

int ret;

/*

* The strncpy_from_kernel_nofault() call will likely not fill the

* entire buffer, but that's okay in this circumstance as we're probing

* arbitrary memory anyway similar to bpf_probe_read_*() and might

* as well probe the stack. Thus, memory is explicitly cleared

* only in error case, so that improper users ignoring return

* code altogether don't copy garbage; otherwise length of string

* is returned that can be used for bpf_perf_event_output() et al.

*/

ret = strncpy_from_kernel_nofault(dst, unsafe_ptr, size);

if (unlikely(ret < 0))

memset(dst, 0, size);

return ret;

}

BPF_CALL_3(bpf_probe_read_kernel_str, void *, dst, u32, size,

const void *, unsafe_ptr)

{

return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr);

}

const struct bpf_func_proto bpf_probe_read_kernel_str_proto = {

.func = bpf_probe_read_kernel_str,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_UNINIT_MEM,

.arg2_type = ARG_CONST_SIZE_OR_ZERO,

.arg3_type = ARG_ANYTHING,

};

#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE

BPF_CALL_3(bpf_probe_read_compat, void *, dst, u32, size,

const void *, unsafe_ptr)

{

if ((unsigned long)unsafe_ptr < TASK_SIZE) {

return bpf_probe_read_user_common(dst, size,

(__force void __user *)unsafe_ptr);

}

return bpf_probe_read_kernel_common(dst, size, unsafe_ptr);

}

static const struct bpf_func_proto bpf_probe_read_compat_proto = {

.func = bpf_probe_read_compat,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_UNINIT_MEM,

.arg2_type = ARG_CONST_SIZE_OR_ZERO,

.arg3_type = ARG_ANYTHING,

};

BPF_CALL_3(bpf_probe_read_compat_str, void *, dst, u32, size,

const void *, unsafe_ptr)

{

if ((unsigned long)unsafe_ptr < TASK_SIZE) {

return bpf_probe_read_user_str_common(dst, size,

(__force void __user *)unsafe_ptr);

}

return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr);

}

static const struct bpf_func_proto bpf_probe_read_compat_str_proto = {

.func = bpf_probe_read_compat_str,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_UNINIT_MEM,

.arg2_type = ARG_CONST_SIZE_OR_ZERO,

.arg3_type = ARG_ANYTHING,

};

#endif /* CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE */

BPF_CALL_3(bpf_probe_write_user, void __user *, unsafe_ptr, const void *, src,

u32, size)

{

/*

* Ensure we're in user context which is safe for the helper to

* run. This helper has no business in a kthread.

*

* access_ok() should prevent writing to non-user memory, but in

* some situations (nommu, temporary switch, etc) access_ok() does

* not provide enough validation, hence the check on KERNEL_DS.

*

* nmi_uaccess_okay() ensures the probe is not run in an interim

* state, when the task or mm are switched. This is specifically

* required to prevent the use of temporary mm.

*/

if (unlikely(in_interrupt() ||

current->flags & (PF_KTHREAD | PF_EXITING)))

return -EPERM;

if (unlikely(!nmi_uaccess_okay()))

return -EPERM;

return copy_to_user_nofault(unsafe_ptr, src, size);

}

static const struct bpf_func_proto bpf_probe_write_user_proto = {

.func = bpf_probe_write_user,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_ANYTHING,

.arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,

.arg3_type = ARG_CONST_SIZE,

};

#define MAX_TRACE_PRINTK_VARARGS 3

#define BPF_TRACE_PRINTK_SIZE 1024

BPF_CALL_5(bpf_trace_printk, char *, fmt, u32, fmt_size, u64, arg1,

u64, arg2, u64, arg3)

{

u64 args[MAX_TRACE_PRINTK_VARARGS] = { arg1, arg2, arg3 };

struct bpf_bprintf_data data = {

.get_bin_args = true,

.get_buf = true,

};

int ret;

ret = bpf_bprintf_prepare(fmt, fmt_size, args,

MAX_TRACE_PRINTK_VARARGS, &data);

if (ret < 0)

return ret;

ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt, data.bin_args);

trace_bpf_trace_printk(data.buf);

bpf_bprintf_cleanup(&data);

return ret;

}

static const struct bpf_func_proto bpf_trace_printk_proto = {

.func = bpf_trace_printk,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,

.arg2_type = ARG_CONST_SIZE,

};

static void __set_printk_clr_event(struct work_struct *work)

{

/*

* This program might be calling bpf_trace_printk,

* so enable the associated bpf_trace/bpf_trace_printk event.

* Repeat this each time as it is possible a user has

* disabled bpf_trace_printk events. By loading a program

* calling bpf_trace_printk() however the user has expressed

* the intent to see such events.

*/

if (trace_set_clr_event("bpf_trace", "bpf_trace_printk", 1))

pr_warn_ratelimited("could not enable bpf_trace_printk events");

}

static DECLARE_WORK(set_printk_work, __set_printk_clr_event);

const struct bpf_func_proto *bpf_get_trace_printk_proto(void)

{

schedule_work(&set_printk_work);

return &bpf_trace_printk_proto;

}

BPF_CALL_4(bpf_trace_vprintk, char *, fmt, u32, fmt_size, const void *, args,

u32, data_len)

{

struct bpf_bprintf_data data = {

.get_bin_args = true,

.get_buf = true,

};

int ret, num_args;

if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 ||

(data_len && !args))

return -EINVAL;

num_args = data_len / 8;

ret = bpf_bprintf_prepare(fmt, fmt_size, args, num_args, &data);

if (ret < 0)

return ret;

ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt, data.bin_args);

trace_bpf_trace_printk(data.buf);

bpf_bprintf_cleanup(&data);

return ret;

}

static const struct bpf_func_proto bpf_trace_vprintk_proto = {

.func = bpf_trace_vprintk,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,

.arg2_type = ARG_CONST_SIZE,

.arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,

.arg4_type = ARG_CONST_SIZE_OR_ZERO,

};

const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void)

{

schedule_work(&set_printk_work);

return &bpf_trace_vprintk_proto;

}

BPF_CALL_5(bpf_seq_printf, struct seq_file *, m, char *, fmt, u32, fmt_size,

const void *, args, u32, data_len)

{

struct bpf_bprintf_data data = {

.get_bin_args = true,

};

int err, num_args;

if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 ||

(data_len && !args))

return -EINVAL;

num_args = data_len / 8;

err = bpf_bprintf_prepare(fmt, fmt_size, args, num_args, &data);

if (err < 0)

return err;

seq_bprintf(m, fmt, data.bin_args);

bpf_bprintf_cleanup(&data);

return seq_has_overflowed(m) ? -EOVERFLOW : 0;

}

BTF_ID_LIST_SINGLE(btf_seq_file_ids, struct, seq_file)

static const struct bpf_func_proto bpf_seq_printf_proto = {

.func = bpf_seq_printf,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_BTF_ID,

.arg1_btf_id = &btf_seq_file_ids[0],

.arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,

.arg3_type = ARG_CONST_SIZE,

.arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,

.arg5_type = ARG_CONST_SIZE_OR_ZERO,

};

BPF_CALL_3(bpf_seq_write, struct seq_file *, m, const void *, data, u32, len)

{

return seq_write(m, data, len) ? -EOVERFLOW : 0;

}

static const struct bpf_func_proto bpf_seq_write_proto = {

.func = bpf_seq_write,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_BTF_ID,

.arg1_btf_id = &btf_seq_file_ids[0],

.arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,

.arg3_type = ARG_CONST_SIZE_OR_ZERO,

};

BPF_CALL_4(bpf_seq_printf_btf, struct seq_file *, m, struct btf_ptr *, ptr,

u32, btf_ptr_size, u64, flags)

{

const struct btf *btf;

s32 btf_id;

int ret;

ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id);

if (ret)

return ret;

return btf_type_seq_show_flags(btf, btf_id, ptr->ptr, m, flags);

}

static const struct bpf_func_proto bpf_seq_printf_btf_proto = {

.func = bpf_seq_printf_btf,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_BTF_ID,

.arg1_btf_id = &btf_seq_file_ids[0],

.arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,

.arg3_type = ARG_CONST_SIZE_OR_ZERO,

.arg4_type = ARG_ANYTHING,

};

static __always_inline int

get_map_perf_counter(struct bpf_map *map, u64 flags,

u64 *value, u64 *enabled, u64 *running)

{

struct bpf_array *array = container_of(map, struct bpf_array, map);

unsigned int cpu = smp_processor_id();

u64 index = flags & BPF_F_INDEX_MASK;

struct bpf_event_entry *ee;

if (unlikely(flags & ~(BPF_F_INDEX_MASK)))

return -EINVAL;

if (index == BPF_F_CURRENT_CPU)

index = cpu;

if (unlikely(index >= array->map.max_entries))

return -E2BIG;

ee = READ_ONCE(array->ptrs[index]);

if (!ee)

return -ENOENT;

return perf_event_read_local(ee->event, value, enabled, running);

}

BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags)

{

u64 value = 0;

int err;

err = get_map_perf_counter(map, flags, &value, NULL, NULL);

/*

* this api is ugly since we miss [-22..-2] range of valid

* counter values, but that's uapi

*/

if (err)

return err;

return value;

}

const struct bpf_func_proto bpf_perf_event_read_proto = {

.func = bpf_perf_event_read,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_CONST_MAP_PTR,

.arg2_type = ARG_ANYTHING,

};

BPF_CALL_4(bpf_perf_event_read_value, struct bpf_map *, map, u64, flags,

struct bpf_perf_event_value *, buf, u32, size)

{

int err = -EINVAL;

if (unlikely(size != sizeof(struct bpf_perf_event_value)))

goto clear;

err = get_map_perf_counter(map, flags, &buf->counter, &buf->enabled,

&buf->running);

if (unlikely(err))

goto clear;

return 0;

clear:

memset(buf, 0, size);

return err;

}

static const struct bpf_func_proto bpf_perf_event_read_value_proto = {

.func = bpf_perf_event_read_value,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_CONST_MAP_PTR,

.arg2_type = ARG_ANYTHING,

.arg3_type = ARG_PTR_TO_UNINIT_MEM,

.arg4_type = ARG_CONST_SIZE,

};

const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void)

{

return &bpf_perf_event_read_value_proto;

}

static __always_inline u64

__bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,

u64 flags, struct perf_raw_record *raw,

struct perf_sample_data *sd)

{

struct bpf_array *array = container_of(map, struct bpf_array, map);

unsigned int cpu = smp_processor_id();

u64 index = flags & BPF_F_INDEX_MASK;

struct bpf_event_entry *ee;

struct perf_event *event;

if (index == BPF_F_CURRENT_CPU)

index = cpu;

if (unlikely(index >= array->map.max_entries))

return -E2BIG;

ee = READ_ONCE(array->ptrs[index]);

if (!ee)

return -ENOENT;

event = ee->event;

if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE ||

event->attr.config != PERF_COUNT_SW_BPF_OUTPUT))

return -EINVAL;

if (unlikely(event->oncpu != cpu))

return -EOPNOTSUPP;

perf_sample_save_raw_data(sd, event, raw);

return perf_event_output(event, sd, regs);

}

/*

* Support executing tracepoints in normal, irq, and nmi context that each call

* bpf_perf_event_output

*/

struct bpf_trace_sample_data {

struct perf_sample_data sds[3];

};

static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds);

static DEFINE_PER_CPU(int, bpf_trace_nest_level);

BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,

u64, flags, void *, data, u64, size)

{

struct bpf_trace_sample_data *sds;

struct perf_raw_record raw = {

.frag = {

.size = size,

.data = data,

},

};

struct perf_sample_data *sd;

int nest_level, err;

preempt_disable();

sds = this_cpu_ptr(&bpf_trace_sds);

nest_level = this_cpu_inc_return(bpf_trace_nest_level);

if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(sds->sds))) {

err = -EBUSY;

goto out;

}

sd = &sds->sds[nest_level - 1];

if (unlikely(flags & ~(BPF_F_INDEX_MASK))) {

err = -EINVAL;

goto out;

}

perf_sample_data_init(sd, 0, 0);

err = __bpf_perf_event_output(regs, map, flags, &raw, sd);

out:

this_cpu_dec(bpf_trace_nest_level);

preempt_enable();

return err;

}

static const struct bpf_func_proto bpf_perf_event_output_proto = {

.func = bpf_perf_event_output,

.gpl_only = true,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_CTX,

.arg2_type = ARG_CONST_MAP_PTR,

.arg3_type = ARG_ANYTHING,

.arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,

.arg5_type = ARG_CONST_SIZE_OR_ZERO,

};

static DEFINE_PER_CPU(int, bpf_event_output_nest_level);

struct bpf_nested_pt_regs {

struct pt_regs regs[3];

};

static DEFINE_PER_CPU(struct bpf_nested_pt_regs, bpf_pt_regs);

static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_misc_sds);

u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,

void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)

{

struct perf_raw_frag frag = {

.copy = ctx_copy,

.size = ctx_size,

.data = ctx,

};

struct perf_raw_record raw = {

.frag = {

{

.next = ctx_size ? &frag : NULL,

},

.size = meta_size,

.data = meta,

},

};

struct perf_sample_data *sd;

struct pt_regs *regs;

int nest_level;

u64 ret;

preempt_disable();

nest_level = this_cpu_inc_return(bpf_event_output_nest_level);

if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bpf_misc_sds.sds))) {

ret = -EBUSY;

goto out;

}

sd = this_cpu_ptr(&bpf_misc_sds.sds[nest_level - 1]);

regs = this_cpu_ptr(&bpf_pt_regs.regs[nest_level - 1]);

perf_fetch_caller_regs(regs);

perf_sample_data_init(sd, 0, 0);

ret = __bpf_perf_event_output(regs, map, flags, &raw, sd);

out:

this_cpu_dec(bpf_event_output_nest_level);

preempt_enable();

return ret;

}

BPF_CALL_0(bpf_get_current_task)

{

return (long) current;

}

const struct bpf_func_proto bpf_get_current_task_proto = {

.func = bpf_get_current_task,

.gpl_only = true,

.ret_type = RET_INTEGER,

};

BPF_CALL_0(bpf_get_current_task_btf)

{

return (unsigned long) current;

}

const struct bpf_func_proto bpf_get_current_task_btf_proto = {

.func = bpf_get_current_task_btf,

.gpl_only = true,

.ret_type = RET_PTR_TO_BTF_ID_TRUSTED,

.ret_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],

};

BPF_CALL_1(bpf_task_pt_regs, struct task_struct *, task)

{

return (unsigned long) task_pt_regs(task);

}

BTF_ID_LIST_SINGLE(bpf_task_pt_regs_ids, struct, pt_regs)

const struct bpf_func_proto bpf_task_pt_regs_proto = {

.func = bpf_task_pt_regs,

.gpl_only = true,

.arg1_type = ARG_PTR_TO_BTF_ID,

.arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],

.ret_type = RET_PTR_TO_BTF_ID,

.ret_btf_id = &bpf_task_pt_regs_ids[0],

};

struct send_signal_irq_work {

struct irq_work irq_work;

struct task_struct *task;

u32 sig;

enum pid_type type;

bool has_siginfo;

struct kernel_siginfo info;

};

static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work);

static void do_bpf_send_signal(struct irq_work *entry)

{

struct send_signal_irq_work *work;

struct kernel_siginfo *siginfo;

work = container_of(entry, struct send_signal_irq_work, irq_work);

siginfo = work->has_siginfo ? &work->info : SEND_SIG_PRIV;

group_send_sig_info(work->sig, siginfo, work->task, work->type);

put_task_struct(work->task);

}

static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struct *task, u64 value)

{

struct send_signal_irq_work *work = NULL;

struct kernel_siginfo info;

struct kernel_siginfo *siginfo;

if (!task) {

task = current;

siginfo = SEND_SIG_PRIV;

} else {

clear_siginfo(&info);

info.si_signo = sig;

info.si_errno = 0;

info.si_code = SI_KERNEL;

info.si_pid = 0;

info.si_uid = 0;

info.si_value.sival_ptr = (void __user __force *)(unsigned long)value;

siginfo = &info;

}

/* Similar to bpf_probe_write_user, task needs to be

* in a sound condition and kernel memory access be

* permitted in order to send signal to the current

* task.

*/

if (unlikely(task->flags & (PF_KTHREAD | PF_EXITING)))

return -EPERM;

if (unlikely(!nmi_uaccess_okay()))

return -EPERM;

/* Task should not be pid=1 to avoid kernel panic. */

if (unlikely(is_global_init(task)))

return -EPERM;

if (preempt_count() != 0 || irqs_disabled()) {

/* Do an early check on signal validity. Otherwise,

* the error is lost in deferred irq_work.

*/

if (unlikely(!valid_signal(sig)))

return -EINVAL;

work = this_cpu_ptr(&send_signal_work);

if (irq_work_is_busy(&work->irq_work))

return -EBUSY;

/* Add the current task, which is the target of sending signal,

* to the irq_work. The current task may change when queued

* irq works get executed.

*/

work->task = get_task_struct(task);

work->has_siginfo = siginfo == &info;

if (work->has_siginfo)

copy_siginfo(&work->info, &info);

work->sig = sig;

work->type = type;

irq_work_queue(&work->irq_work);

return 0;

}

return group_send_sig_info(sig, siginfo, task, type);

}

BPF_CALL_1(bpf_send_signal, u32, sig)

{

return bpf_send_signal_common(sig, PIDTYPE_TGID, NULL, 0);

}

const struct bpf_func_proto bpf_send_signal_proto = {

.func = bpf_send_signal,

.gpl_only = false,

.ret_type = RET_INTEGER,

.arg1_type = ARG_ANYTHING,

};

BPF_CALL_1(bpf_send_signal_thread, u32, sig)

{

return bpf_send_signal_common(sig, PIDTYPE_PID, NULL, 0);

}

const struct bpf_func_proto bpf_send_signal_thread_proto = {

.func = bpf_send_signal_thread,

.gpl_only = false,

.ret_type = RET_INTEGER,

.arg1_type = ARG_ANYTHING,

};

BPF_CALL_3(bpf_d_path, const struct path *, path, char *, buf, u32, sz)

{

struct path copy;

long len;

char *p;

if (!sz)

return 0;

/*

* The path pointer is verified as trusted and safe to use,

* but let's double check it's valid anyway to workaround

* potentially broken verifier.

*/

len = copy_from_kernel_nofault(&copy, path, sizeof(*path));

if (len < 0)

return len;

p = d_path(&copy, buf, sz);

if (IS_ERR(p)) {

len = PTR_ERR(p);

} else {

len = buf + sz - p;

memmove(buf, p, len);

}

return len;

}

BTF_SET_START(btf_allowlist_d_path)

#ifdef CONFIG_SECURITY

BTF_ID(func, security_file_permission)

BTF_ID(func, security_inode_getattr)

BTF_ID(func, security_file_open)

#endif

#ifdef CONFIG_SECURITY_PATH

BTF_ID(func, security_path_truncate)

#endif

BTF_ID(func, vfs_truncate)

BTF_ID(func, vfs_fallocate)

BTF_ID(func, dentry_open)

BTF_ID(func, vfs_getattr)

BTF_ID(func, filp_close)

BTF_SET_END(btf_allowlist_d_path)

static bool bpf_d_path_allowed(const struct bpf_prog *prog)

{

if (prog->type == BPF_PROG_TYPE_TRACING &&

prog->expected_attach_type == BPF_TRACE_ITER)

return true;

if (prog->type == BPF_PROG_TYPE_LSM)

return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id);

return btf_id_set_contains(&btf_allowlist_d_path,

prog->aux->attach_btf_id);

}

BTF_ID_LIST_SINGLE(bpf_d_path_btf_ids, struct, path)

static const struct bpf_func_proto bpf_d_path_proto = {

.func = bpf_d_path,

.gpl_only = false,

.ret_type = RET_INTEGER,

.arg1_type = ARG_PTR_TO_BTF_ID,

.arg1_btf_id = &bpf_d_path_btf_ids[0],

.arg2_type = ARG_PTR_TO_MEM | MEM_WRITE,

.arg3_type = ARG_CONST_SIZE_OR_ZERO,

.allowed = bpf_d_path_allowed,