// SPDX-License-Identifier: GPL-2.0-only

/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com

*/

#include <crypto/sha2.h>

#include <linux/bpf.h>

#include <linux/bpf-cgroup.h>

#include <linux/bpf_trace.h>

#include <linux/bpf_lirc.h>

#include <linux/bpf_verifier.h>

#include <linux/bsearch.h>

#include <linux/btf.h>

#include <linux/hex.h>

#include <linux/syscalls.h>

#include <linux/slab.h>

#include <linux/sched/signal.h>

#include <linux/vmalloc.h>

#include <linux/mmzone.h>

#include <linux/anon_inodes.h>

#include <linux/fdtable.h>

#include <linux/file.h>

#include <linux/fs.h>

#include <linux/license.h>

#include <linux/filter.h>

#include <linux/kernel.h>

#include <linux/idr.h>

#include <linux/cred.h>

#include <linux/timekeeping.h>

#include <linux/ctype.h>

#include <linux/nospec.h>

#include <linux/audit.h>

#include <uapi/linux/btf.h>

#include <linux/pgtable.h>

#include <linux/bpf_lsm.h>

#include <linux/poll.h>

#include <linux/sort.h>

#include <linux/bpf-netns.h>

#include <linux/rcupdate_trace.h>

#include <linux/memcontrol.h>

#include <linux/trace_events.h>

#include <linux/tracepoint.h>

#include <linux/overflow.h>

#include <linux/cookie.h>

#include <linux/verification.h>

#include <net/netfilter/nf_bpf_link.h>

#include <net/netkit.h>

#include <net/tcx.h>

#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \

(map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \

(map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)

#define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)

#define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)

#define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \

IS_FD_HASH(map))

#define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY)

DEFINE_PER_CPU(int, bpf_prog_active);

DEFINE_COOKIE(bpf_map_cookie);

static DEFINE_IDR(prog_idr);

static DEFINE_SPINLOCK(prog_idr_lock);

static DEFINE_IDR(map_idr);

static DEFINE_SPINLOCK(map_idr_lock);

static DEFINE_IDR(link_idr);

static DEFINE_SPINLOCK(link_idr_lock);

int sysctl_unprivileged_bpf_disabled __read_mostly =

IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0;

static const struct bpf_map_ops * const bpf_map_types[] = {

#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)

#define BPF_MAP_TYPE(_id, _ops) \

[_id] = &_ops,

#define BPF_LINK_TYPE(_id, _name)

#include <linux/bpf_types.h>

#undef BPF_PROG_TYPE

#undef BPF_MAP_TYPE

#undef BPF_LINK_TYPE

};

/*

* If we're handed a bigger struct than we know of, ensure all the unknown bits

* are 0 - i.e. new user-space does not rely on any kernel feature extensions

* we don't know about yet.

*

* There is a ToCToU between this function call and the following

* copy_from_user() call. However, this is not a concern since this function is

* meant to be a future-proofing of bits.

*/

int bpf_check_uarg_tail_zero(bpfptr_t uaddr,

size_t expected_size,

size_t actual_size)

{

int res;

if (unlikely(actual_size > PAGE_SIZE)) /* silly large */

return -E2BIG;

if (actual_size <= expected_size)

return 0;

if (uaddr.is_kernel)

res = memchr_inv(uaddr.kernel + expected_size, 0,

actual_size - expected_size) == NULL;

else

res = check_zeroed_user(uaddr.user + expected_size,

actual_size - expected_size);

if (res < 0)

return res;

return res ? 0 : -E2BIG;

}

const struct bpf_map_ops bpf_map_offload_ops = {

.map_meta_equal = bpf_map_meta_equal,

.map_alloc = bpf_map_offload_map_alloc,

.map_free = bpf_map_offload_map_free,

.map_check_btf = map_check_no_btf,

.map_mem_usage = bpf_map_offload_map_mem_usage,

};

static void bpf_map_write_active_inc(struct bpf_map *map)

{

atomic64_inc(&map->writecnt);

}

static void bpf_map_write_active_dec(struct bpf_map *map)

{

atomic64_dec(&map->writecnt);

}

bool bpf_map_write_active(const struct bpf_map *map)

{

return atomic64_read(&map->writecnt) != 0;

}

static u32 bpf_map_value_size(const struct bpf_map *map, u64 flags)

{

if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS))

return map->value_size;

else if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||

map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||

map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||

map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)

return round_up(map->value_size, 8) * num_possible_cpus();

else if (IS_FD_MAP(map))

return sizeof(u32);

else

return map->value_size;

}

static void maybe_wait_bpf_programs(struct bpf_map *map)

{

/* Wait for any running non-sleepable BPF programs to complete so that

* userspace, when we return to it, knows that all non-sleepable

* programs that could be running use the new map value. For sleepable

* BPF programs, synchronize_rcu_tasks_trace() should be used to wait

* for the completions of these programs, but considering the waiting

* time can be very long and userspace may think it will hang forever,

* so don't handle sleepable BPF programs now.

*/

if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||

map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)

synchronize_rcu_expedited();

}

static void unpin_uptr_kaddr(void *kaddr)

{

if (kaddr)

unpin_user_page(virt_to_page(kaddr));

}

static void __bpf_obj_unpin_uptrs(struct btf_record *rec, u32 cnt, void *obj)

{

const struct btf_field *field;

void **uptr_addr;

int i;

for (i = 0, field = rec->fields; i < cnt; i++, field++) {

if (field->type != BPF_UPTR)

continue;

uptr_addr = obj + field->offset;

unpin_uptr_kaddr(*uptr_addr);

}

}

static void bpf_obj_unpin_uptrs(struct btf_record *rec, void *obj)

{

if (!btf_record_has_field(rec, BPF_UPTR))

return;

__bpf_obj_unpin_uptrs(rec, rec->cnt, obj);

}

static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj)

{

const struct btf_field *field;

const struct btf_type *t;

unsigned long start, end;

struct page *page;

void **uptr_addr;

int i, err;

if (!btf_record_has_field(rec, BPF_UPTR))

return 0;

for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) {

if (field->type != BPF_UPTR)

continue;

uptr_addr = obj + field->offset;

start = *(unsigned long *)uptr_addr;

if (!start)

continue;

t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id);

/* t->size was checked for zero before */

if (check_add_overflow(start, t->size - 1, &end)) {

err = -EFAULT;

goto unpin_all;

}

/* The uptr's struct cannot span across two pages */

if ((start & PAGE_MASK) != (end & PAGE_MASK)) {

err = -EOPNOTSUPP;

goto unpin_all;

}

err = pin_user_pages_fast(start, 1, FOLL_LONGTERM | FOLL_WRITE, &page);

if (err != 1)

goto unpin_all;

if (PageHighMem(page)) {

err = -EOPNOTSUPP;

unpin_user_page(page);

goto unpin_all;

}

*uptr_addr = page_address(page) + offset_in_page(start);

}

return 0;

unpin_all:

__bpf_obj_unpin_uptrs(rec, i, obj);

return err;

}

static int bpf_map_update_value(struct bpf_map *map, struct file *map_file,

void *key, void *value, __u64 flags)

{

int err;

/* Need to create a kthread, thus must support schedule */

if (bpf_map_is_offloaded(map)) {

return bpf_map_offload_update_elem(map, key, value, flags);

} else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||

map->map_type == BPF_MAP_TYPE_ARENA ||

map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {

return map->ops->map_update_elem(map, key, value, flags);

} else if (map->map_type == BPF_MAP_TYPE_SOCKHASH ||

map->map_type == BPF_MAP_TYPE_SOCKMAP) {

return sock_map_update_elem_sys(map, key, value, flags);

} else if (IS_FD_PROG_ARRAY(map)) {

return bpf_fd_array_map_update_elem(map, map_file, key, value,

flags);

}

bpf_disable_instrumentation();

if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||

map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {

err = bpf_percpu_hash_update(map, key, value, flags);

} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {

err = bpf_percpu_array_update(map, key, value, flags);

} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {

err = bpf_percpu_cgroup_storage_update(map, key, value,

flags);

} else if (IS_FD_ARRAY(map)) {

err = bpf_fd_array_map_update_elem(map, map_file, key, value,

flags);

} else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {

err = bpf_fd_htab_map_update_elem(map, map_file, key, value,

flags);

} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {

/* rcu_read_lock() is not needed */

err = bpf_fd_reuseport_array_update_elem(map, key, value,

flags);

} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||

map->map_type == BPF_MAP_TYPE_STACK ||

map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {

err = map->ops->map_push_elem(map, value, flags);

} else {

err = bpf_obj_pin_uptrs(map->record, value);

if (!err) {

rcu_read_lock();

err = map->ops->map_update_elem(map, key, value, flags);

rcu_read_unlock();

if (err)

bpf_obj_unpin_uptrs(map->record, value);

}

}

bpf_enable_instrumentation();

return err;

}

static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,

__u64 flags)

{

void *ptr;

int err;

if (bpf_map_is_offloaded(map))

return bpf_map_offload_lookup_elem(map, key, value);

bpf_disable_instrumentation();

if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||

map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {

err = bpf_percpu_hash_copy(map, key, value, flags);

} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {

err = bpf_percpu_array_copy(map, key, value, flags);

} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {

err = bpf_percpu_cgroup_storage_copy(map, key, value, flags);

} else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {

err = bpf_stackmap_extract(map, key, value, false);

} else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {

err = bpf_fd_array_map_lookup_elem(map, key, value);

} else if (IS_FD_HASH(map)) {

err = bpf_fd_htab_map_lookup_elem(map, key, value);

} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {

err = bpf_fd_reuseport_array_lookup_elem(map, key, value);

} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||

map->map_type == BPF_MAP_TYPE_STACK ||

map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {

err = map->ops->map_peek_elem(map, value);

} else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {

/* struct_ops map requires directly updating "value" */

err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);

} else {

rcu_read_lock();

if (map->ops->map_lookup_elem_sys_only)

ptr = map->ops->map_lookup_elem_sys_only(map, key);

else

ptr = map->ops->map_lookup_elem(map, key);

if (IS_ERR(ptr)) {

err = PTR_ERR(ptr);

} else if (!ptr) {

err = -ENOENT;

} else {

err = 0;

if (flags & BPF_F_LOCK)

/* lock 'ptr' and copy everything but lock */

copy_map_value_locked(map, value, ptr, true);

else

copy_map_value(map, value, ptr);

/* mask lock and timer, since value wasn't zero inited */

check_and_init_map_value(map, value);

}

rcu_read_unlock();

}

bpf_enable_instrumentation();

return err;

}

/* Please, do not use this function outside from the map creation path

* (e.g. in map update path) without taking care of setting the active

* memory cgroup (see at bpf_map_kmalloc_node() for example).

*/

static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)

{

/* We really just want to fail instead of triggering OOM killer

* under memory pressure, therefore we set __GFP_NORETRY to kmalloc,

* which is used for lower order allocation requests.

*

* It has been observed that higher order allocation requests done by

* vmalloc with __GFP_NORETRY being set might fail due to not trying

* to reclaim memory from the page cache, thus we set

* __GFP_RETRY_MAYFAIL to avoid such situations.

*/

gfp_t gfp = bpf_memcg_flags(__GFP_NOWARN | __GFP_ZERO);

unsigned int flags = 0;

unsigned long align = 1;

void *area;

if (size >= SIZE_MAX)

return NULL;

/* kmalloc()'ed memory can't be mmap()'ed */

if (mmapable) {

BUG_ON(!PAGE_ALIGNED(size));

align = SHMLBA;

flags = VM_USERMAP;

} else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {

area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY,

numa_node);

if (area != NULL)

return area;

}

return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,

gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL,

flags, numa_node, __builtin_return_address(0));

}

void *bpf_map_area_alloc(u64 size, int numa_node)

{

return __bpf_map_area_alloc(size, numa_node, false);

}

void *bpf_map_area_mmapable_alloc(u64 size, int numa_node)

{

return __bpf_map_area_alloc(size, numa_node, true);

}

void bpf_map_area_free(void *area)

{

kvfree(area);

}

static u32 bpf_map_flags_retain_permanent(u32 flags)

{

/* Some map creation flags are not tied to the map object but

* rather to the map fd instead, so they have no meaning upon

* map object inspection since multiple file descriptors with

* different (access) properties can exist here. Thus, given

* this has zero meaning for the map itself, lets clear these

* from here.

*/

return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);

}

void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)

{

map->map_type = attr->map_type;

map->key_size = attr->key_size;

map->value_size = attr->value_size;

map->max_entries = attr->max_entries;

map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);

map->numa_node = bpf_map_attr_numa_node(attr);

map->map_extra = attr->map_extra;

}

static int bpf_map_alloc_id(struct bpf_map *map)

{

int id;

idr_preload(GFP_KERNEL);

spin_lock_bh(&map_idr_lock);

id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);

if (id > 0)

map->id = id;

spin_unlock_bh(&map_idr_lock);

idr_preload_end();

if (WARN_ON_ONCE(!id))

return -ENOSPC;

return id > 0 ? 0 : id;

}

void bpf_map_free_id(struct bpf_map *map)

{

unsigned long flags;

/* Offloaded maps are removed from the IDR store when their device

* disappears - even if someone holds an fd to them they are unusable,

* the memory is gone, all ops will fail; they are simply waiting for

* refcnt to drop to be freed.

*/

if (!map->id)

return;

spin_lock_irqsave(&map_idr_lock, flags);

idr_remove(&map_idr, map->id);

map->id = 0;

spin_unlock_irqrestore(&map_idr_lock, flags);

}

#ifdef CONFIG_MEMCG

static void bpf_map_save_memcg(struct bpf_map *map)

{

/* Currently if a map is created by a process belonging to the root

* memory cgroup, get_obj_cgroup_from_current() will return NULL.

* So we have to check map->objcg for being NULL each time it's

* being used.

*/

if (memcg_bpf_enabled())

map->objcg = get_obj_cgroup_from_current();

}

static void bpf_map_release_memcg(struct bpf_map *map)

{

if (map->objcg)

obj_cgroup_put(map->objcg);

}

static struct mem_cgroup *bpf_map_get_memcg(const struct bpf_map *map)

{

if (map->objcg)

return get_mem_cgroup_from_objcg(map->objcg);

return root_mem_cgroup;

}

void bpf_map_memcg_enter(const struct bpf_map *map, struct mem_cgroup **old_memcg,

struct mem_cgroup **new_memcg)

{

*new_memcg = bpf_map_get_memcg(map);

*old_memcg = set_active_memcg(*new_memcg);

}

void bpf_map_memcg_exit(struct mem_cgroup *old_memcg,

struct mem_cgroup *new_memcg)

{

set_active_memcg(old_memcg);

mem_cgroup_put(new_memcg);

}

void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,

int node)

{

struct mem_cgroup *memcg, *old_memcg;

void *ptr;

bpf_map_memcg_enter(map, &old_memcg, &memcg);

ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node);

bpf_map_memcg_exit(old_memcg, memcg);

return ptr;

}

void *bpf_map_kmalloc_nolock(const struct bpf_map *map, size_t size, gfp_t flags,

int node)

{

struct mem_cgroup *memcg, *old_memcg;

void *ptr;

bpf_map_memcg_enter(map, &old_memcg, &memcg);

ptr = kmalloc_nolock(size, flags | __GFP_ACCOUNT, node);

bpf_map_memcg_exit(old_memcg, memcg);

return ptr;

}

void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)

{

struct mem_cgroup *memcg, *old_memcg;

void *ptr;

bpf_map_memcg_enter(map, &old_memcg, &memcg);

ptr = kzalloc(size, flags | __GFP_ACCOUNT);

bpf_map_memcg_exit(old_memcg, memcg);

return ptr;

}

void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,

gfp_t flags)

{

struct mem_cgroup *memcg, *old_memcg;

void *ptr;

bpf_map_memcg_enter(map, &old_memcg, &memcg);

ptr = kvcalloc(n, size, flags | __GFP_ACCOUNT);

bpf_map_memcg_exit(old_memcg, memcg);

return ptr;

}

void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,

size_t align, gfp_t flags)

{

struct mem_cgroup *memcg, *old_memcg;

void __percpu *ptr;

bpf_map_memcg_enter(map, &old_memcg, &memcg);

ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT);

bpf_map_memcg_exit(old_memcg, memcg);

return ptr;

}

#else

static void bpf_map_save_memcg(struct bpf_map *map)

{

}

static void bpf_map_release_memcg(struct bpf_map *map)

{

}

#endif

static bool can_alloc_pages(void)

{

return preempt_count() == 0 && !irqs_disabled() &&

!IS_ENABLED(CONFIG_PREEMPT_RT);

}

static struct page *__bpf_alloc_page(int nid)

{

if (!can_alloc_pages())

return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0);

return alloc_pages_node(nid,

GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT

| __GFP_NOWARN,

0);

}

int bpf_map_alloc_pages(const struct bpf_map *map, int nid,

unsigned long nr_pages, struct page **pages)

{

unsigned long i, j;

struct page *pg;

int ret = 0;

for (i = 0; i < nr_pages; i++) {

pg = __bpf_alloc_page(nid);

if (pg) {

pages[i] = pg;

continue;

}

for (j = 0; j < i; j++)

free_pages_nolock(pages[j], 0);

ret = -ENOMEM;

break;

}

return ret;

}

static int btf_field_cmp(const void *a, const void *b)

{

const struct btf_field *f1 = a, *f2 = b;

if (f1->offset < f2->offset)

return -1;

else if (f1->offset > f2->offset)

return 1;

return 0;

}

struct btf_field *btf_record_find(const struct btf_record *rec, u32 offset,

u32 field_mask)

{

struct btf_field *field;

if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & field_mask))

return NULL;

field = bsearch(&offset, rec->fields, rec->cnt, sizeof(rec->fields[0]), btf_field_cmp);

if (!field || !(field->type & field_mask))

return NULL;

return field;

}

void btf_record_free(struct btf_record *rec)

{

int i;

if (IS_ERR_OR_NULL(rec))

return;

for (i = 0; i < rec->cnt; i++) {

switch (rec->fields[i].type) {

case BPF_KPTR_UNREF:

case BPF_KPTR_REF:

case BPF_KPTR_PERCPU:

case BPF_UPTR:

if (rec->fields[i].kptr.module)

module_put(rec->fields[i].kptr.module);

if (btf_is_kernel(rec->fields[i].kptr.btf))

btf_put(rec->fields[i].kptr.btf);

break;

case BPF_LIST_HEAD:

case BPF_LIST_NODE:

case BPF_RB_ROOT:

case BPF_RB_NODE:

case BPF_SPIN_LOCK:

case BPF_RES_SPIN_LOCK:

case BPF_TIMER:

case BPF_REFCOUNT:

case BPF_WORKQUEUE:

case BPF_TASK_WORK:

/* Nothing to release */

break;

default:

WARN_ON_ONCE(1);

continue;

}

}

kfree(rec);

}

void bpf_map_free_record(struct bpf_map *map)

{

btf_record_free(map->record);

map->record = NULL;

}

struct btf_record *btf_record_dup(const struct btf_record *rec)

{

const struct btf_field *fields;

struct btf_record *new_rec;

int ret, size, i;

if (IS_ERR_OR_NULL(rec))

return NULL;

size = struct_size(rec, fields, rec->cnt);

new_rec = kmemdup(rec, size, GFP_KERNEL | __GFP_NOWARN);

if (!new_rec)

return ERR_PTR(-ENOMEM);

/* Do a deep copy of the btf_record */

fields = rec->fields;

new_rec->cnt = 0;

for (i = 0; i < rec->cnt; i++) {

switch (fields[i].type) {

case BPF_KPTR_UNREF:

case BPF_KPTR_REF:

case BPF_KPTR_PERCPU:

case BPF_UPTR:

if (btf_is_kernel(fields[i].kptr.btf))

btf_get(fields[i].kptr.btf);

if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) {

ret = -ENXIO;

goto free;

}

break;

case BPF_LIST_HEAD:

case BPF_LIST_NODE:

case BPF_RB_ROOT:

case BPF_RB_NODE:

case BPF_SPIN_LOCK:

case BPF_RES_SPIN_LOCK:

case BPF_TIMER:

case BPF_REFCOUNT:

case BPF_WORKQUEUE:

case BPF_TASK_WORK:

/* Nothing to acquire */

break;

default:

ret = -EFAULT;

WARN_ON_ONCE(1);

goto free;

}

new_rec->cnt++;

}

return new_rec;

free:

btf_record_free(new_rec);

return ERR_PTR(ret);

}

bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b)

{

bool a_has_fields = !IS_ERR_OR_NULL(rec_a), b_has_fields = !IS_ERR_OR_NULL(rec_b);

int size;

if (!a_has_fields && !b_has_fields)

return true;

if (a_has_fields != b_has_fields)

return false;

if (rec_a->cnt != rec_b->cnt)

return false;

size = struct_size(rec_a, fields, rec_a->cnt);

/* btf_parse_fields uses kzalloc to allocate a btf_record, so unused

* members are zeroed out. So memcmp is safe to do without worrying

* about padding/unused fields.

*

* While spin_lock, timer, and kptr have no relation to map BTF,

* list_head metadata is specific to map BTF, the btf and value_rec

* members in particular. btf is the map BTF, while value_rec points to

* btf_record in that map BTF.

*

* So while by default, we don't rely on the map BTF (which the records

* were parsed from) matching for both records, which is not backwards

* compatible, in case list_head is part of it, we implicitly rely on

* that by way of depending on memcmp succeeding for it.

*/

return !memcmp(rec_a, rec_b, size);

}

void bpf_obj_free_timer(const struct btf_record *rec, void *obj)

{

if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_TIMER)))

return;

bpf_timer_cancel_and_free(obj + rec->timer_off);

}

void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj)

{

if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_WORKQUEUE)))

return;

bpf_wq_cancel_and_free(obj + rec->wq_off);

}

void bpf_obj_free_task_work(const struct btf_record *rec, void *obj)

{

if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_TASK_WORK)))

return;

bpf_task_work_cancel_and_free(obj + rec->task_work_off);

}

void bpf_obj_free_fields(const struct btf_record *rec, void *obj)

{

const struct btf_field *fields;

int i;

if (IS_ERR_OR_NULL(rec))

return;

fields = rec->fields;

for (i = 0; i < rec->cnt; i++) {

struct btf_struct_meta *pointee_struct_meta;

const struct btf_field *field = &fields[i];

void *field_ptr = obj + field->offset;

void *xchgd_field;

switch (fields[i].type) {

case BPF_SPIN_LOCK:

case BPF_RES_SPIN_LOCK:

break;

case BPF_TIMER:

bpf_timer_cancel_and_free(field_ptr);

break;

case BPF_WORKQUEUE:

bpf_wq_cancel_and_free(field_ptr);

break;

case BPF_TASK_WORK:

bpf_task_work_cancel_and_free(field_ptr);

break;

case BPF_KPTR_UNREF:

WRITE_ONCE(*(u64 *)field_ptr, 0);

break;

case BPF_KPTR_REF:

case BPF_KPTR_PERCPU:

xchgd_field = (void *)xchg((unsigned long *)field_ptr, 0);

if (!xchgd_field)

break;

if (!btf_is_kernel(field->kptr.btf)) {

pointee_struct_meta = btf_find_struct_meta(field->kptr.btf,

field->kptr.btf_id);

__bpf_obj_drop_impl(xchgd_field, pointee_struct_meta ?

pointee_struct_meta->record : NULL,

fields[i].type == BPF_KPTR_PERCPU);

} else {

field->kptr.dtor(xchgd_field);

}

break;

case BPF_UPTR:

/* The caller ensured that no one is using the uptr */

unpin_uptr_kaddr(*(void **)field_ptr);

break;

case BPF_LIST_HEAD:

if (WARN_ON_ONCE(rec->spin_lock_off < 0))

continue;

bpf_list_head_free(field, field_ptr, obj + rec->spin_lock_off);

break;

case BPF_RB_ROOT:

if (WARN_ON_ONCE(rec->spin_lock_off < 0))

continue;

bpf_rb_root_free(field, field_ptr, obj + rec->spin_lock_off);

break;

case BPF_LIST_NODE:

case BPF_RB_NODE:

case BPF_REFCOUNT:

break;

default:

WARN_ON_ONCE(1);

continue;

}

}

}

static void bpf_map_free(struct bpf_map *map)

{

struct btf_record *rec = map->record;

struct btf *btf = map->btf;

/* implementation dependent freeing. Disabling migration to simplify

* the free of values or special fields allocated from bpf memory

* allocator.

*/

kfree(map->excl_prog_sha);

migrate_disable();

map->ops->map_free(map);

migrate_enable();

/* Delay freeing of btf_record for maps, as map_free

* callback usually needs access to them. It is better to do it here

* than require each callback to do the free itself manually.

*

* Note that the btf_record stashed in map->inner_map_meta->record was

* already freed using the map_free callback for map in map case which

* eventually calls bpf_map_free_meta, since inner_map_meta is only a

* template bpf_map struct used during verification.

*/

btf_record_free(rec);

/* Delay freeing of btf for maps, as map_free callback may need

* struct_meta info which will be freed with btf_put().

*/

btf_put(btf);

}

/* called from workqueue */

static void bpf_map_free_deferred(struct work_struct *work)

{

struct bpf_map *map = container_of(work, struct bpf_map, work);

security_bpf_map_free(map);

bpf_map_release_memcg(map);

bpf_map_owner_free(map);

bpf_map_free(map);

}

static void bpf_map_put_uref(struct bpf_map *map)

{

if (atomic64_dec_and_test(&map->usercnt)) {

if (map->ops->map_release_uref)

map->ops->map_release_uref(map);

}

}

static void bpf_map_free_in_work(struct bpf_map *map)

{

INIT_WORK(&map->work, bpf_map_free_deferred);

/* Avoid spawning kworkers, since they all might contend

* for the same mutex like slab_mutex.

*/

queue_work(system_dfl_wq, &map->work);

}

static void bpf_map_free_rcu_gp(struct rcu_head *rcu)

{

bpf_map_free_in_work(container_of(rcu, struct bpf_map, rcu));

}

static void bpf_map_free_mult_rcu_gp(struct rcu_head *rcu)

{

if (rcu_trace_implies_rcu_gp())

bpf_map_free_rcu_gp(rcu);

else

call_rcu(rcu, bpf_map_free_rcu_gp);

}

/* decrement map refcnt and schedule it for freeing via workqueue

* (underlying map implementation ops->map_free() might sleep)

*/

void bpf_map_put(struct bpf_map *map)

{

if (atomic64_dec_and_test(&map->refcnt)) {

/* bpf_map_free_id() must be called first */

bpf_map_free_id(map);

WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt));

if (READ_ONCE(map->free_after_mult_rcu_gp))

call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp);

else if (READ_ONCE(map->free_after_rcu_gp))

call_rcu(&map->rcu, bpf_map_free_rcu_gp);

else

bpf_map_free_in_work(map);

}

}

EXPORT_SYMBOL_GPL(bpf_map_put);

void bpf_map_put_with_uref(struct bpf_map *map)

{

bpf_map_put_uref(map);

bpf_map_put(map);

}

static int bpf_map_release(struct inode *inode, struct file *filp)

{

struct bpf_map *map = filp->private_data;

if (map->ops->map_release)

map->ops->map_release(map, filp);

bpf_map_put_with_uref(map);

return 0;

}

static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)

{

fmode_t mode = fd_file(f)->f_mode;

/* Our file permissions may have been overridden by global

* map permissions facing syscall side.

*/

if (READ_ONCE(map->frozen))

mode &= ~FMODE_CAN_WRITE;

return mode;

}