1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2019 Facebook */
3
4#include <linux/bpf.h>
5#include <linux/bpf_verifier.h>
6#include <linux/btf.h>
7#include <linux/filter.h>
8#include <linux/slab.h>
9#include <linux/numa.h>
10#include <linux/seq_file.h>
11#include <linux/refcount.h>
12#include <linux/mutex.h>
13#include <linux/btf_ids.h>
14#include <linux/rcupdate_wait.h>
15#include <linux/poll.h>
16
17struct bpf_struct_ops_value {
18 struct bpf_struct_ops_common_value common;
19 char data[] ____cacheline_aligned_in_smp;
20};
21
22#define MAX_TRAMP_IMAGE_PAGES 8
23
24struct bpf_struct_ops_map {
25 struct bpf_map map;
26 const struct bpf_struct_ops_desc *st_ops_desc;
27 /* protect map_update */
28 struct mutex lock;
29 /* link has all the bpf_links that is populated
30 * to the func ptr of the kernel's struct
31 * (in kvalue.data).
32 */
33 struct bpf_link **links;
34 /* ksyms for bpf trampolines */
35 struct bpf_ksym **ksyms;
36 u32 funcs_cnt;
37 u32 image_pages_cnt;
38 /* image_pages is an array of pages that has all the trampolines
39 * that stores the func args before calling the bpf_prog.
40 */
41 void *image_pages[MAX_TRAMP_IMAGE_PAGES];
42 /* The owner moduler's btf. */
43 struct btf *btf;
44 /* uvalue->data stores the kernel struct
45 * (e.g. tcp_congestion_ops) that is more useful
46 * to userspace than the kvalue. For example,
47 * the bpf_prog's id is stored instead of the kernel
48 * address of a func ptr.
49 */
50 struct bpf_struct_ops_value *uvalue;
51 /* kvalue.data stores the actual kernel's struct
52 * (e.g. tcp_congestion_ops) that will be
53 * registered to the kernel subsystem.
54 */
55 struct bpf_struct_ops_value kvalue;
56};
57
58struct bpf_struct_ops_link {
59 struct bpf_link link;
60 struct bpf_map __rcu *map;
61 wait_queue_head_t wait_hup;
62};
63
64static DEFINE_MUTEX(update_mutex);
65
66#define VALUE_PREFIX "bpf_struct_ops_"
67#define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1)
68
69const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = {
70};
71
72const struct bpf_prog_ops bpf_struct_ops_prog_ops = {
73#ifdef CONFIG_NET
74 .test_run = bpf_struct_ops_test_run,
75#endif
76};
77
78BTF_ID_LIST(st_ops_ids)
79BTF_ID(struct, module)
80BTF_ID(struct, bpf_struct_ops_common_value)
81
82enum {
83 IDX_MODULE_ID,
84 IDX_ST_OPS_COMMON_VALUE_ID,
85};
86
87extern struct btf *btf_vmlinux;
88
89static bool is_valid_value_type(struct btf *btf, s32 value_id,
90 const struct btf_type *type,
91 const char *value_name)
92{
93 const struct btf_type *common_value_type;
94 const struct btf_member *member;
95 const struct btf_type *vt, *mt;
96
97 vt = btf_type_by_id(btf, type_id: value_id);
98 if (btf_vlen(t: vt) != 2) {
99 pr_warn("The number of %s's members should be 2, but we get %d\n",
100 value_name, btf_vlen(vt));
101 return false;
102 }
103 member = btf_type_member(t: vt);
104 mt = btf_type_by_id(btf, type_id: member->type);
105 common_value_type = btf_type_by_id(btf: btf_vmlinux,
106 type_id: st_ops_ids[IDX_ST_OPS_COMMON_VALUE_ID]);
107 if (mt != common_value_type) {
108 pr_warn("The first member of %s should be bpf_struct_ops_common_value\n",
109 value_name);
110 return false;
111 }
112 member++;
113 mt = btf_type_by_id(btf, type_id: member->type);
114 if (mt != type) {
115 pr_warn("The second member of %s should be %s\n",
116 value_name, btf_name_by_offset(btf, type->name_off));
117 return false;
118 }
119
120 return true;
121}
122
123static void *bpf_struct_ops_image_alloc(void)
124{
125 void *image;
126 int err;
127
128 err = bpf_jit_charge_modmem(PAGE_SIZE);
129 if (err)
130 return ERR_PTR(error: err);
131 image = arch_alloc_bpf_trampoline(PAGE_SIZE);
132 if (!image) {
133 bpf_jit_uncharge_modmem(PAGE_SIZE);
134 return ERR_PTR(error: -ENOMEM);
135 }
136
137 return image;
138}
139
140void bpf_struct_ops_image_free(void *image)
141{
142 if (image) {
143 arch_free_bpf_trampoline(image, PAGE_SIZE);
144 bpf_jit_uncharge_modmem(PAGE_SIZE);
145 }
146}
147
148#define MAYBE_NULL_SUFFIX "__nullable"
149#define REFCOUNTED_SUFFIX "__ref"
150
151/* Prepare argument info for every nullable argument of a member of a
152 * struct_ops type.
153 *
154 * Initialize a struct bpf_struct_ops_arg_info according to type info of
155 * the arguments of a stub function. (Check kCFI for more information about
156 * stub functions.)
157 *
158 * Each member in the struct_ops type has a struct bpf_struct_ops_arg_info
159 * to provide an array of struct bpf_ctx_arg_aux, which in turn provides
160 * the information that used by the verifier to check the arguments of the
161 * BPF struct_ops program assigned to the member. Here, we only care about
162 * the arguments that are marked as __nullable.
163 *
164 * The array of struct bpf_ctx_arg_aux is eventually assigned to
165 * prog->aux->ctx_arg_info of BPF struct_ops programs and passed to the
166 * verifier. (See check_struct_ops_btf_id())
167 *
168 * arg_info->info will be the list of struct bpf_ctx_arg_aux if success. If
169 * fails, it will be kept untouched.
170 */
171static int prepare_arg_info(struct btf *btf,
172 const char *st_ops_name,
173 const char *member_name,
174 const struct btf_type *func_proto, void *stub_func_addr,
175 struct bpf_struct_ops_arg_info *arg_info)
176{
177 const struct btf_type *stub_func_proto, *pointed_type;
178 bool is_nullable = false, is_refcounted = false;
179 const struct btf_param *stub_args, *args;
180 struct bpf_ctx_arg_aux *info, *info_buf;
181 u32 nargs, arg_no, info_cnt = 0;
182 char ksym[KSYM_SYMBOL_LEN];
183 const char *stub_fname;
184 const char *suffix;
185 s32 stub_func_id;
186 u32 arg_btf_id;
187 int offset;
188
189 stub_fname = kallsyms_lookup(addr: (unsigned long)stub_func_addr, NULL, NULL, NULL, namebuf: ksym);
190 if (!stub_fname) {
191 pr_warn("Cannot find the stub function name for the %s in struct %s\n",
192 member_name, st_ops_name);
193 return -ENOENT;
194 }
195
196 stub_func_id = btf_find_by_name_kind(btf, name: stub_fname, kind: BTF_KIND_FUNC);
197 if (stub_func_id < 0) {
198 pr_warn("Cannot find the stub function %s in btf\n", stub_fname);
199 return -ENOENT;
200 }
201
202 stub_func_proto = btf_type_by_id(btf, type_id: stub_func_id);
203 stub_func_proto = btf_type_by_id(btf, type_id: stub_func_proto->type);
204
205 /* Check if the number of arguments of the stub function is the same
206 * as the number of arguments of the function pointer.
207 */
208 nargs = btf_type_vlen(t: func_proto);
209 if (nargs != btf_type_vlen(t: stub_func_proto)) {
210 pr_warn("the number of arguments of the stub function %s does not match the number of arguments of the member %s of struct %s\n",
211 stub_fname, member_name, st_ops_name);
212 return -EINVAL;
213 }
214
215 if (!nargs)
216 return 0;
217
218 args = btf_params(t: func_proto);
219 stub_args = btf_params(t: stub_func_proto);
220
221 info_buf = kcalloc(nargs, sizeof(*info_buf), GFP_KERNEL);
222 if (!info_buf)
223 return -ENOMEM;
224
225 /* Prepare info for every nullable argument */
226 info = info_buf;
227 for (arg_no = 0; arg_no < nargs; arg_no++) {
228 /* Skip arguments that is not suffixed with
229 * "__nullable or __ref".
230 */
231 is_nullable = btf_param_match_suffix(btf, arg: &stub_args[arg_no],
232 MAYBE_NULL_SUFFIX);
233 is_refcounted = btf_param_match_suffix(btf, arg: &stub_args[arg_no],
234 REFCOUNTED_SUFFIX);
235
236 if (is_nullable)
237 suffix = MAYBE_NULL_SUFFIX;
238 else if (is_refcounted)
239 suffix = REFCOUNTED_SUFFIX;
240 else
241 continue;
242
243 /* Should be a pointer to struct */
244 pointed_type = btf_type_resolve_ptr(btf,
245 id: args[arg_no].type,
246 res_id: &arg_btf_id);
247 if (!pointed_type ||
248 !btf_type_is_struct(t: pointed_type)) {
249 pr_warn("stub function %s has %s tagging to an unsupported type\n",
250 stub_fname, suffix);
251 goto err_out;
252 }
253
254 offset = btf_ctx_arg_offset(btf, func_proto, arg_no);
255 if (offset < 0) {
256 pr_warn("stub function %s has an invalid trampoline ctx offset for arg#%u\n",
257 stub_fname, arg_no);
258 goto err_out;
259 }
260
261 if (args[arg_no].type != stub_args[arg_no].type) {
262 pr_warn("arg#%u type in stub function %s does not match with its original func_proto\n",
263 arg_no, stub_fname);
264 goto err_out;
265 }
266
267 /* Fill the information of the new argument */
268 info->btf_id = arg_btf_id;
269 info->btf = btf;
270 info->offset = offset;
271 if (is_nullable) {
272 info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID | PTR_MAYBE_NULL;
273 } else if (is_refcounted) {
274 info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID;
275 info->refcounted = true;
276 }
277
278 info++;
279 info_cnt++;
280 }
281
282 if (info_cnt) {
283 arg_info->info = info_buf;
284 arg_info->cnt = info_cnt;
285 } else {
286 kfree(objp: info_buf);
287 }
288
289 return 0;
290
291err_out:
292 kfree(objp: info_buf);
293
294 return -EINVAL;
295}
296
297/* Clean up the arg_info in a struct bpf_struct_ops_desc. */
298void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc)
299{
300 struct bpf_struct_ops_arg_info *arg_info;
301 int i;
302
303 arg_info = st_ops_desc->arg_info;
304 for (i = 0; i < btf_type_vlen(t: st_ops_desc->type); i++)
305 kfree(objp: arg_info[i].info);
306
307 kfree(objp: arg_info);
308}
309
310static bool is_module_member(const struct btf *btf, u32 id)
311{
312 const struct btf_type *t;
313
314 t = btf_type_resolve_ptr(btf, id, NULL);
315 if (!t)
316 return false;
317
318 if (!__btf_type_is_struct(t) && !btf_type_is_fwd(t))
319 return false;
320
321 return !strcmp(btf_name_by_offset(btf, offset: t->name_off), "module");
322}
323
324int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)
325{
326 void *func_ptr = *(void **)(st_ops->cfi_stubs + moff);
327
328 return func_ptr ? 0 : -ENOTSUPP;
329}
330
331int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
332 struct btf *btf,
333 struct bpf_verifier_log *log)
334{
335 struct bpf_struct_ops *st_ops = st_ops_desc->st_ops;
336 struct bpf_struct_ops_arg_info *arg_info;
337 const struct btf_member *member;
338 const struct btf_type *t;
339 s32 type_id, value_id;
340 char value_name[128];
341 const char *mname;
342 int i, err;
343
344 if (strlen(st_ops->name) + VALUE_PREFIX_LEN >=
345 sizeof(value_name)) {
346 pr_warn("struct_ops name %s is too long\n",
347 st_ops->name);
348 return -EINVAL;
349 }
350 sprintf(buf: value_name, fmt: "%s%s", VALUE_PREFIX, st_ops->name);
351
352 if (!st_ops->cfi_stubs) {
353 pr_warn("struct_ops for %s has no cfi_stubs\n", st_ops->name);
354 return -EINVAL;
355 }
356
357 type_id = btf_find_by_name_kind(btf, name: st_ops->name,
358 kind: BTF_KIND_STRUCT);
359 if (type_id < 0) {
360 pr_warn("Cannot find struct %s in %s\n",
361 st_ops->name, btf_get_name(btf));
362 return -EINVAL;
363 }
364 t = btf_type_by_id(btf, type_id);
365 if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) {
366 pr_warn("Cannot support #%u members in struct %s\n",
367 btf_type_vlen(t), st_ops->name);
368 return -EINVAL;
369 }
370
371 value_id = btf_find_by_name_kind(btf, name: value_name,
372 kind: BTF_KIND_STRUCT);
373 if (value_id < 0) {
374 pr_warn("Cannot find struct %s in %s\n",
375 value_name, btf_get_name(btf));
376 return -EINVAL;
377 }
378 if (!is_valid_value_type(btf, value_id, type: t, value_name))
379 return -EINVAL;
380
381 arg_info = kcalloc(btf_type_vlen(t), sizeof(*arg_info),
382 GFP_KERNEL);
383 if (!arg_info)
384 return -ENOMEM;
385
386 st_ops_desc->arg_info = arg_info;
387 st_ops_desc->type = t;
388 st_ops_desc->type_id = type_id;
389 st_ops_desc->value_id = value_id;
390 st_ops_desc->value_type = btf_type_by_id(btf, type_id: value_id);
391
392 for_each_member(i, t, member) {
393 const struct btf_type *func_proto, *ret_type;
394 void **stub_func_addr;
395 u32 moff;
396
397 moff = __btf_member_bit_offset(struct_type: t, member) / 8;
398 mname = btf_name_by_offset(btf, offset: member->name_off);
399 if (!*mname) {
400 pr_warn("anon member in struct %s is not supported\n",
401 st_ops->name);
402 err = -EOPNOTSUPP;
403 goto errout;
404 }
405
406 if (__btf_member_bitfield_size(struct_type: t, member)) {
407 pr_warn("bit field member %s in struct %s is not supported\n",
408 mname, st_ops->name);
409 err = -EOPNOTSUPP;
410 goto errout;
411 }
412
413 if (!st_ops_ids[IDX_MODULE_ID] && is_module_member(btf, id: member->type)) {
414 pr_warn("'struct module' btf id not found. Is CONFIG_MODULES enabled? bpf_struct_ops '%s' needs module support.\n",
415 st_ops->name);
416 err = -EOPNOTSUPP;
417 goto errout;
418 }
419
420 func_proto = btf_type_resolve_func_ptr(btf,
421 id: member->type,
422 NULL);
423
424 /* The member is not a function pointer or
425 * the function pointer is not supported.
426 */
427 if (!func_proto || bpf_struct_ops_supported(st_ops, moff))
428 continue;
429
430 if (func_proto->type) {
431 ret_type = btf_type_resolve_ptr(btf, id: func_proto->type, NULL);
432 if (ret_type && !__btf_type_is_struct(t: ret_type)) {
433 pr_warn("func ptr %s in struct %s returns non-struct pointer, which is not supported\n",
434 mname, st_ops->name);
435 err = -EOPNOTSUPP;
436 goto errout;
437 }
438 }
439
440 if (btf_distill_func_proto(log, btf,
441 func_proto, func_name: mname,
442 m: &st_ops->func_models[i])) {
443 pr_warn("Error in parsing func ptr %s in struct %s\n",
444 mname, st_ops->name);
445 err = -EINVAL;
446 goto errout;
447 }
448
449 stub_func_addr = *(void **)(st_ops->cfi_stubs + moff);
450 err = prepare_arg_info(btf, st_ops_name: st_ops->name, member_name: mname,
451 func_proto, stub_func_addr,
452 arg_info: arg_info + i);
453 if (err)
454 goto errout;
455 }
456
457 if (st_ops->init(btf)) {
458 pr_warn("Error in init bpf_struct_ops %s\n",
459 st_ops->name);
460 err = -EINVAL;
461 goto errout;
462 }
463
464 return 0;
465
466errout:
467 bpf_struct_ops_desc_release(st_ops_desc);
468
469 return err;
470}
471
472static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key,
473 void *next_key)
474{
475 if (key && *(u32 *)key == 0)
476 return -ENOENT;
477
478 *(u32 *)next_key = 0;
479 return 0;
480}
481
482int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
483 void *value)
484{
485 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
486 struct bpf_struct_ops_value *uvalue, *kvalue;
487 enum bpf_struct_ops_state state;
488 s64 refcnt;
489
490 if (unlikely(*(u32 *)key != 0))
491 return -ENOENT;
492
493 kvalue = &st_map->kvalue;
494 /* Pair with smp_store_release() during map_update */
495 state = smp_load_acquire(&kvalue->common.state);
496 if (state == BPF_STRUCT_OPS_STATE_INIT) {
497 memset(value, 0, map->value_size);
498 return 0;
499 }
500
501 /* No lock is needed. state and refcnt do not need
502 * to be updated together under atomic context.
503 */
504 uvalue = value;
505 memcpy(uvalue, st_map->uvalue, map->value_size);
506 uvalue->common.state = state;
507
508 /* This value offers the user space a general estimate of how
509 * many sockets are still utilizing this struct_ops for TCP
510 * congestion control. The number might not be exact, but it
511 * should sufficiently meet our present goals.
512 */
513 refcnt = atomic64_read(v: &map->refcnt) - atomic64_read(v: &map->usercnt);
514 refcount_set(r: &uvalue->common.refcnt, max_t(s64, refcnt, 0));
515
516 return 0;
517}
518
519static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key)
520{
521 return ERR_PTR(error: -EINVAL);
522}
523
524static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)
525{
526 u32 i;
527
528 for (i = 0; i < st_map->funcs_cnt; i++) {
529 if (!st_map->links[i])
530 break;
531 bpf_link_put(link: st_map->links[i]);
532 st_map->links[i] = NULL;
533 }
534}
535
536static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map)
537{
538 int i;
539
540 for (i = 0; i < st_map->image_pages_cnt; i++)
541 bpf_struct_ops_image_free(image: st_map->image_pages[i]);
542 st_map->image_pages_cnt = 0;
543}
544
545static int check_zero_holes(const struct btf *btf, const struct btf_type *t, void *data)
546{
547 const struct btf_member *member;
548 u32 i, moff, msize, prev_mend = 0;
549 const struct btf_type *mtype;
550
551 for_each_member(i, t, member) {
552 moff = __btf_member_bit_offset(struct_type: t, member) / 8;
553 if (moff > prev_mend &&
554 memchr_inv(p: data + prev_mend, c: 0, size: moff - prev_mend))
555 return -EINVAL;
556
557 mtype = btf_type_by_id(btf, type_id: member->type);
558 mtype = btf_resolve_size(btf, type: mtype, type_size: &msize);
559 if (IS_ERR(ptr: mtype))
560 return PTR_ERR(ptr: mtype);
561 prev_mend = moff + msize;
562 }
563
564 if (t->size > prev_mend &&
565 memchr_inv(p: data + prev_mend, c: 0, size: t->size - prev_mend))
566 return -EINVAL;
567
568 return 0;
569}
570
571static void bpf_struct_ops_link_release(struct bpf_link *link)
572{
573}
574
575static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
576{
577 struct bpf_tramp_link *tlink = container_of(link, struct bpf_tramp_link, link);
578
579 kfree(objp: tlink);
580}
581
582const struct bpf_link_ops bpf_struct_ops_link_lops = {
583 .release = bpf_struct_ops_link_release,
584 .dealloc = bpf_struct_ops_link_dealloc,
585};
586
587int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
588 struct bpf_tramp_link *link,
589 const struct btf_func_model *model,
590 void *stub_func,
591 void **_image, u32 *_image_off,
592 bool allow_alloc)
593{
594 u32 image_off = *_image_off, flags = BPF_TRAMP_F_INDIRECT;
595 void *image = *_image;
596 int size;
597
598 tlinks[BPF_TRAMP_FENTRY].links[0] = link;
599 tlinks[BPF_TRAMP_FENTRY].nr_links = 1;
600
601 if (model->ret_size > 0)
602 flags |= BPF_TRAMP_F_RET_FENTRY_RET;
603
604 size = arch_bpf_trampoline_size(m: model, flags, tlinks, func_addr: stub_func);
605 if (size <= 0)
606 return size ? : -EFAULT;
607
608 /* Allocate image buffer if necessary */
609 if (!image || size > PAGE_SIZE - image_off) {
610 if (!allow_alloc)
611 return -E2BIG;
612
613 image = bpf_struct_ops_image_alloc();
614 if (IS_ERR(ptr: image))
615 return PTR_ERR(ptr: image);
616 image_off = 0;
617 }
618
619 size = arch_prepare_bpf_trampoline(NULL, image: image + image_off,
620 image_end: image + image_off + size,
621 m: model, flags, tlinks, func_addr: stub_func);
622 if (size <= 0) {
623 if (image != *_image)
624 bpf_struct_ops_image_free(image);
625 return size ? : -EFAULT;
626 }
627
628 *_image = image;
629 *_image_off = image_off + size;
630 return 0;
631}
632
633static void bpf_struct_ops_ksym_init(const char *tname, const char *mname,
634 void *image, unsigned int size,
635 struct bpf_ksym *ksym)
636{
637 snprintf(buf: ksym->name, KSYM_NAME_LEN, fmt: "bpf__%s_%s", tname, mname);
638 INIT_LIST_HEAD_RCU(list: &ksym->lnode);
639 bpf_image_ksym_init(data: image, size, ksym);
640}
641
642static void bpf_struct_ops_map_add_ksyms(struct bpf_struct_ops_map *st_map)
643{
644 u32 i;
645
646 for (i = 0; i < st_map->funcs_cnt; i++) {
647 if (!st_map->ksyms[i])
648 break;
649 bpf_image_ksym_add(ksym: st_map->ksyms[i]);
650 }
651}
652
653static void bpf_struct_ops_map_del_ksyms(struct bpf_struct_ops_map *st_map)
654{
655 u32 i;
656
657 for (i = 0; i < st_map->funcs_cnt; i++) {
658 if (!st_map->ksyms[i])
659 break;
660 bpf_image_ksym_del(ksym: st_map->ksyms[i]);
661 }
662}
663
664static void bpf_struct_ops_map_free_ksyms(struct bpf_struct_ops_map *st_map)
665{
666 u32 i;
667
668 for (i = 0; i < st_map->funcs_cnt; i++) {
669 if (!st_map->ksyms[i])
670 break;
671 kfree(objp: st_map->ksyms[i]);
672 st_map->ksyms[i] = NULL;
673 }
674}
675
676static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
677 void *value, u64 flags)
678{
679 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
680 const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;
681 const struct bpf_struct_ops *st_ops = st_ops_desc->st_ops;
682 struct bpf_struct_ops_value *uvalue, *kvalue;
683 const struct btf_type *module_type;
684 const struct btf_member *member;
685 const struct btf_type *t = st_ops_desc->type;
686 struct bpf_tramp_links *tlinks;
687 void *udata, *kdata;
688 int prog_fd, err;
689 u32 i, trampoline_start, image_off = 0;
690 void *cur_image = NULL, *image = NULL;
691 struct bpf_link **plink;
692 struct bpf_ksym **pksym;
693 const char *tname, *mname;
694
695 if (flags)
696 return -EINVAL;
697
698 if (*(u32 *)key != 0)
699 return -E2BIG;
700
701 err = check_zero_holes(btf: st_map->btf, t: st_ops_desc->value_type, data: value);
702 if (err)
703 return err;
704
705 uvalue = value;
706 err = check_zero_holes(btf: st_map->btf, t, data: uvalue->data);
707 if (err)
708 return err;
709
710 if (uvalue->common.state || refcount_read(r: &uvalue->common.refcnt))
711 return -EINVAL;
712
713 tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
714 if (!tlinks)
715 return -ENOMEM;
716
717 uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
718 kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;
719
720 mutex_lock(&st_map->lock);
721
722 if (kvalue->common.state != BPF_STRUCT_OPS_STATE_INIT) {
723 err = -EBUSY;
724 goto unlock;
725 }
726
727 memcpy(uvalue, value, map->value_size);
728
729 udata = &uvalue->data;
730 kdata = &kvalue->data;
731
732 plink = st_map->links;
733 pksym = st_map->ksyms;
734 tname = btf_name_by_offset(btf: st_map->btf, offset: t->name_off);
735 module_type = btf_type_by_id(btf: btf_vmlinux, type_id: st_ops_ids[IDX_MODULE_ID]);
736 for_each_member(i, t, member) {
737 const struct btf_type *mtype, *ptype;
738 struct bpf_prog *prog;
739 struct bpf_tramp_link *link;
740 struct bpf_ksym *ksym;
741 u32 moff;
742
743 moff = __btf_member_bit_offset(struct_type: t, member) / 8;
744 mname = btf_name_by_offset(btf: st_map->btf, offset: member->name_off);
745 ptype = btf_type_resolve_ptr(btf: st_map->btf, id: member->type, NULL);
746 if (ptype == module_type) {
747 if (*(void **)(udata + moff))
748 goto reset_unlock;
749 *(void **)(kdata + moff) = BPF_MODULE_OWNER;
750 continue;
751 }
752
753 err = st_ops->init_member(t, member, kdata, udata);
754 if (err < 0)
755 goto reset_unlock;
756
757 /* The ->init_member() has handled this member */
758 if (err > 0)
759 continue;
760
761 /* If st_ops->init_member does not handle it,
762 * we will only handle func ptrs and zero-ed members
763 * here. Reject everything else.
764 */
765
766 /* All non func ptr member must be 0 */
767 if (!ptype || !btf_type_is_func_proto(t: ptype)) {
768 u32 msize;
769
770 mtype = btf_type_by_id(btf: st_map->btf, type_id: member->type);
771 mtype = btf_resolve_size(btf: st_map->btf, type: mtype, type_size: &msize);
772 if (IS_ERR(ptr: mtype)) {
773 err = PTR_ERR(ptr: mtype);
774 goto reset_unlock;
775 }
776
777 if (memchr_inv(p: udata + moff, c: 0, size: msize)) {
778 err = -EINVAL;
779 goto reset_unlock;
780 }
781
782 continue;
783 }
784
785 prog_fd = (int)(*(unsigned long *)(udata + moff));
786 /* Similar check as the attr->attach_prog_fd */
787 if (!prog_fd)
788 continue;
789
790 prog = bpf_prog_get(ufd: prog_fd);
791 if (IS_ERR(ptr: prog)) {
792 err = PTR_ERR(ptr: prog);
793 goto reset_unlock;
794 }
795
796 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
797 prog->aux->attach_btf_id != st_ops_desc->type_id ||
798 prog->expected_attach_type != i) {
799 bpf_prog_put(prog);
800 err = -EINVAL;
801 goto reset_unlock;
802 }
803
804 link = kzalloc(sizeof(*link), GFP_USER);
805 if (!link) {
806 bpf_prog_put(prog);
807 err = -ENOMEM;
808 goto reset_unlock;
809 }
810 bpf_link_init(link: &link->link, type: BPF_LINK_TYPE_STRUCT_OPS,
811 ops: &bpf_struct_ops_link_lops, prog, attach_type: prog->expected_attach_type);
812 *plink++ = &link->link;
813
814 ksym = kzalloc(sizeof(*ksym), GFP_USER);
815 if (!ksym) {
816 err = -ENOMEM;
817 goto reset_unlock;
818 }
819 *pksym++ = ksym;
820
821 trampoline_start = image_off;
822 err = bpf_struct_ops_prepare_trampoline(tlinks, link,
823 model: &st_ops->func_models[i],
824 stub_func: *(void **)(st_ops->cfi_stubs + moff),
825 image: &image, image_off: &image_off,
826 allow_alloc: st_map->image_pages_cnt < MAX_TRAMP_IMAGE_PAGES);
827 if (err)
828 goto reset_unlock;
829
830 if (cur_image != image) {
831 st_map->image_pages[st_map->image_pages_cnt++] = image;
832 cur_image = image;
833 trampoline_start = 0;
834 }
835
836 *(void **)(kdata + moff) = image + trampoline_start + cfi_get_offset();
837
838 /* put prog_id to udata */
839 *(unsigned long *)(udata + moff) = prog->aux->id;
840
841 /* init ksym for this trampoline */
842 bpf_struct_ops_ksym_init(tname, mname,
843 image: image + trampoline_start,
844 size: image_off - trampoline_start,
845 ksym);
846 }
847
848 if (st_ops->validate) {
849 err = st_ops->validate(kdata);
850 if (err)
851 goto reset_unlock;
852 }
853 for (i = 0; i < st_map->image_pages_cnt; i++) {
854 err = arch_protect_bpf_trampoline(image: st_map->image_pages[i],
855 PAGE_SIZE);
856 if (err)
857 goto reset_unlock;
858 }
859
860 if (st_map->map.map_flags & BPF_F_LINK) {
861 err = 0;
862 /* Let bpf_link handle registration & unregistration.
863 *
864 * Pair with smp_load_acquire() during lookup_elem().
865 */
866 smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_READY);
867 goto unlock;
868 }
869
870 err = st_ops->reg(kdata, NULL);
871 if (likely(!err)) {
872 /* This refcnt increment on the map here after
873 * 'st_ops->reg()' is secure since the state of the
874 * map must be set to INIT at this moment, and thus
875 * bpf_struct_ops_map_delete_elem() can't unregister
876 * or transition it to TOBEFREE concurrently.
877 */
878 bpf_map_inc(map);
879 /* Pair with smp_load_acquire() during lookup_elem().
880 * It ensures the above udata updates (e.g. prog->aux->id)
881 * can be seen once BPF_STRUCT_OPS_STATE_INUSE is set.
882 */
883 smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_INUSE);
884 goto unlock;
885 }
886
887 /* Error during st_ops->reg(). Can happen if this struct_ops needs to be
888 * verified as a whole, after all init_member() calls. Can also happen if
889 * there was a race in registering the struct_ops (under the same name) to
890 * a sub-system through different struct_ops's maps.
891 */
892
893reset_unlock:
894 bpf_struct_ops_map_free_ksyms(st_map);
895 bpf_struct_ops_map_free_image(st_map);
896 bpf_struct_ops_map_put_progs(st_map);
897 memset(uvalue, 0, map->value_size);
898 memset(kvalue, 0, map->value_size);
899unlock:
900 kfree(objp: tlinks);
901 mutex_unlock(lock: &st_map->lock);
902 if (!err)
903 bpf_struct_ops_map_add_ksyms(st_map);
904 return err;
905}
906
907static long bpf_struct_ops_map_delete_elem(struct bpf_map *map, void *key)
908{
909 enum bpf_struct_ops_state prev_state;
910 struct bpf_struct_ops_map *st_map;
911
912 st_map = (struct bpf_struct_ops_map *)map;
913 if (st_map->map.map_flags & BPF_F_LINK)
914 return -EOPNOTSUPP;
915
916 prev_state = cmpxchg(&st_map->kvalue.common.state,
917 BPF_STRUCT_OPS_STATE_INUSE,
918 BPF_STRUCT_OPS_STATE_TOBEFREE);
919 switch (prev_state) {
920 case BPF_STRUCT_OPS_STATE_INUSE:
921 st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, NULL);
922 bpf_map_put(map);
923 return 0;
924 case BPF_STRUCT_OPS_STATE_TOBEFREE:
925 return -EINPROGRESS;
926 case BPF_STRUCT_OPS_STATE_INIT:
927 return -ENOENT;
928 default:
929 WARN_ON_ONCE(1);
930 /* Should never happen. Treat it as not found. */
931 return -ENOENT;
932 }
933}
934
935static void bpf_struct_ops_map_seq_show_elem(struct bpf_map *map, void *key,
936 struct seq_file *m)
937{
938 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
939 void *value;
940 int err;
941
942 value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);
943 if (!value)
944 return;
945
946 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
947 if (!err) {
948 btf_type_seq_show(btf: st_map->btf,
949 type_id: map->btf_vmlinux_value_type_id,
950 obj: value, m);
951 seq_putc(m, c: '\n');
952 }
953
954 kfree(objp: value);
955}
956
957static void __bpf_struct_ops_map_free(struct bpf_map *map)
958{
959 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
960
961 if (st_map->links)
962 bpf_struct_ops_map_put_progs(st_map);
963 if (st_map->ksyms)
964 bpf_struct_ops_map_free_ksyms(st_map);
965 bpf_map_area_free(base: st_map->links);
966 bpf_map_area_free(base: st_map->ksyms);
967 bpf_struct_ops_map_free_image(st_map);
968 bpf_map_area_free(base: st_map->uvalue);
969 bpf_map_area_free(base: st_map);
970}
971
972static void bpf_struct_ops_map_free(struct bpf_map *map)
973{
974 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
975
976 /* st_ops->owner was acquired during map_alloc to implicitly holds
977 * the btf's refcnt. The acquire was only done when btf_is_module()
978 * st_map->btf cannot be NULL here.
979 */
980 if (btf_is_module(btf: st_map->btf))
981 module_put(module: st_map->st_ops_desc->st_ops->owner);
982
983 bpf_struct_ops_map_del_ksyms(st_map);
984
985 /* The struct_ops's function may switch to another struct_ops.
986 *
987 * For example, bpf_tcp_cc_x->init() may switch to
988 * another tcp_cc_y by calling
989 * setsockopt(TCP_CONGESTION, "tcp_cc_y").
990 * During the switch, bpf_struct_ops_put(tcp_cc_x) is called
991 * and its refcount may reach 0 which then free its
992 * trampoline image while tcp_cc_x is still running.
993 *
994 * A vanilla rcu gp is to wait for all bpf-tcp-cc prog
995 * to finish. bpf-tcp-cc prog is non sleepable.
996 * A rcu_tasks gp is to wait for the last few insn
997 * in the tramopline image to finish before releasing
998 * the trampoline image.
999 */
1000 synchronize_rcu_mult(call_rcu, call_rcu_tasks);
1001
1002 __bpf_struct_ops_map_free(map);
1003}
1004
1005static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr)
1006{
1007 if (attr->key_size != sizeof(unsigned int) || attr->max_entries != 1 ||
1008 (attr->map_flags & ~(BPF_F_LINK | BPF_F_VTYPE_BTF_OBJ_FD)) ||
1009 !attr->btf_vmlinux_value_type_id)
1010 return -EINVAL;
1011 return 0;
1012}
1013
1014static u32 count_func_ptrs(const struct btf *btf, const struct btf_type *t)
1015{
1016 int i;
1017 u32 count;
1018 const struct btf_member *member;
1019
1020 count = 0;
1021 for_each_member(i, t, member)
1022 if (btf_type_resolve_func_ptr(btf, id: member->type, NULL))
1023 count++;
1024 return count;
1025}
1026
1027static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
1028{
1029 const struct bpf_struct_ops_desc *st_ops_desc;
1030 size_t st_map_size;
1031 struct bpf_struct_ops_map *st_map;
1032 const struct btf_type *t, *vt;
1033 struct module *mod = NULL;
1034 struct bpf_map *map;
1035 struct btf *btf;
1036 int ret;
1037
1038 if (attr->map_flags & BPF_F_VTYPE_BTF_OBJ_FD) {
1039 /* The map holds btf for its whole life time. */
1040 btf = btf_get_by_fd(fd: attr->value_type_btf_obj_fd);
1041 if (IS_ERR(ptr: btf))
1042 return ERR_CAST(ptr: btf);
1043 if (!btf_is_module(btf)) {
1044 btf_put(btf);
1045 return ERR_PTR(error: -EINVAL);
1046 }
1047
1048 mod = btf_try_get_module(btf);
1049 /* mod holds a refcnt to btf. We don't need an extra refcnt
1050 * here.
1051 */
1052 btf_put(btf);
1053 if (!mod)
1054 return ERR_PTR(error: -EINVAL);
1055 } else {
1056 btf = bpf_get_btf_vmlinux();
1057 if (IS_ERR(ptr: btf))
1058 return ERR_CAST(ptr: btf);
1059 if (!btf)
1060 return ERR_PTR(error: -ENOTSUPP);
1061 }
1062
1063 st_ops_desc = bpf_struct_ops_find_value(btf, value_id: attr->btf_vmlinux_value_type_id);
1064 if (!st_ops_desc) {
1065 ret = -ENOTSUPP;
1066 goto errout;
1067 }
1068
1069 vt = st_ops_desc->value_type;
1070 if (attr->value_size != vt->size) {
1071 ret = -EINVAL;
1072 goto errout;
1073 }
1074
1075 t = st_ops_desc->type;
1076
1077 st_map_size = sizeof(*st_map) +
1078 /* kvalue stores the
1079 * struct bpf_struct_ops_tcp_congestions_ops
1080 */
1081 (vt->size - sizeof(struct bpf_struct_ops_value));
1082
1083 st_map = bpf_map_area_alloc(size: st_map_size, NUMA_NO_NODE);
1084 if (!st_map) {
1085 ret = -ENOMEM;
1086 goto errout;
1087 }
1088
1089 st_map->st_ops_desc = st_ops_desc;
1090 map = &st_map->map;
1091
1092 st_map->uvalue = bpf_map_area_alloc(size: vt->size, NUMA_NO_NODE);
1093 st_map->funcs_cnt = count_func_ptrs(btf, t);
1094 st_map->links =
1095 bpf_map_area_alloc(size: st_map->funcs_cnt * sizeof(struct bpf_link *),
1096 NUMA_NO_NODE);
1097
1098 st_map->ksyms =
1099 bpf_map_area_alloc(size: st_map->funcs_cnt * sizeof(struct bpf_ksym *),
1100 NUMA_NO_NODE);
1101 if (!st_map->uvalue || !st_map->links || !st_map->ksyms) {
1102 ret = -ENOMEM;
1103 goto errout_free;
1104 }
1105 st_map->btf = btf;
1106
1107 mutex_init(&st_map->lock);
1108 bpf_map_init_from_attr(map, attr);
1109
1110 return map;
1111
1112errout_free:
1113 __bpf_struct_ops_map_free(map);
1114errout:
1115 module_put(module: mod);
1116
1117 return ERR_PTR(error: ret);
1118}
1119
1120static u64 bpf_struct_ops_map_mem_usage(const struct bpf_map *map)
1121{
1122 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
1123 const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;
1124 const struct btf_type *vt = st_ops_desc->value_type;
1125 u64 usage;
1126
1127 usage = sizeof(*st_map) +
1128 vt->size - sizeof(struct bpf_struct_ops_value);
1129 usage += vt->size;
1130 usage += st_map->funcs_cnt * sizeof(struct bpf_link *);
1131 usage += st_map->funcs_cnt * sizeof(struct bpf_ksym *);
1132 usage += PAGE_SIZE;
1133 return usage;
1134}
1135
1136BTF_ID_LIST_SINGLE(bpf_struct_ops_map_btf_ids, struct, bpf_struct_ops_map)
1137const struct bpf_map_ops bpf_struct_ops_map_ops = {
1138 .map_alloc_check = bpf_struct_ops_map_alloc_check,
1139 .map_alloc = bpf_struct_ops_map_alloc,
1140 .map_free = bpf_struct_ops_map_free,
1141 .map_get_next_key = bpf_struct_ops_map_get_next_key,
1142 .map_lookup_elem = bpf_struct_ops_map_lookup_elem,
1143 .map_delete_elem = bpf_struct_ops_map_delete_elem,
1144 .map_update_elem = bpf_struct_ops_map_update_elem,
1145 .map_seq_show_elem = bpf_struct_ops_map_seq_show_elem,
1146 .map_mem_usage = bpf_struct_ops_map_mem_usage,
1147 .map_btf_id = &bpf_struct_ops_map_btf_ids[0],
1148};
1149
1150/* "const void *" because some subsystem is
1151 * passing a const (e.g. const struct tcp_congestion_ops *)
1152 */
1153bool bpf_struct_ops_get(const void *kdata)
1154{
1155 struct bpf_struct_ops_value *kvalue;
1156 struct bpf_struct_ops_map *st_map;
1157 struct bpf_map *map;
1158
1159 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
1160 st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);
1161
1162 map = __bpf_map_inc_not_zero(map: &st_map->map, uref: false);
1163 return !IS_ERR(ptr: map);
1164}
1165EXPORT_SYMBOL_GPL(bpf_struct_ops_get);
1166
1167void bpf_struct_ops_put(const void *kdata)
1168{
1169 struct bpf_struct_ops_value *kvalue;
1170 struct bpf_struct_ops_map *st_map;
1171
1172 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
1173 st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);
1174
1175 bpf_map_put(map: &st_map->map);
1176}
1177EXPORT_SYMBOL_GPL(bpf_struct_ops_put);
1178
1179u32 bpf_struct_ops_id(const void *kdata)
1180{
1181 struct bpf_struct_ops_value *kvalue;
1182 struct bpf_struct_ops_map *st_map;
1183
1184 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
1185 st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);
1186
1187 return st_map->map.id;
1188}
1189EXPORT_SYMBOL_GPL(bpf_struct_ops_id);
1190
1191static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map)
1192{
1193 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
1194
1195 return map->map_type == BPF_MAP_TYPE_STRUCT_OPS &&
1196 map->map_flags & BPF_F_LINK &&
1197 /* Pair with smp_store_release() during map_update */
1198 smp_load_acquire(&st_map->kvalue.common.state) == BPF_STRUCT_OPS_STATE_READY;
1199}
1200
1201static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link)
1202{
1203 struct bpf_struct_ops_link *st_link;
1204 struct bpf_struct_ops_map *st_map;
1205
1206 st_link = container_of(link, struct bpf_struct_ops_link, link);
1207 st_map = (struct bpf_struct_ops_map *)
1208 rcu_dereference_protected(st_link->map, true);
1209 if (st_map) {
1210 st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link);
1211 bpf_map_put(map: &st_map->map);
1212 }
1213 kfree(objp: st_link);
1214}
1215
1216static void bpf_struct_ops_map_link_show_fdinfo(const struct bpf_link *link,
1217 struct seq_file *seq)
1218{
1219 struct bpf_struct_ops_link *st_link;
1220 struct bpf_map *map;
1221
1222 st_link = container_of(link, struct bpf_struct_ops_link, link);
1223 rcu_read_lock();
1224 map = rcu_dereference(st_link->map);
1225 if (map)
1226 seq_printf(m: seq, fmt: "map_id:\t%d\n", map->id);
1227 rcu_read_unlock();
1228}
1229
1230static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link,
1231 struct bpf_link_info *info)
1232{
1233 struct bpf_struct_ops_link *st_link;
1234 struct bpf_map *map;
1235
1236 st_link = container_of(link, struct bpf_struct_ops_link, link);
1237 rcu_read_lock();
1238 map = rcu_dereference(st_link->map);
1239 if (map)
1240 info->struct_ops.map_id = map->id;
1241 rcu_read_unlock();
1242 return 0;
1243}
1244
1245static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map *new_map,
1246 struct bpf_map *expected_old_map)
1247{
1248 struct bpf_struct_ops_map *st_map, *old_st_map;
1249 struct bpf_map *old_map;
1250 struct bpf_struct_ops_link *st_link;
1251 int err;
1252
1253 st_link = container_of(link, struct bpf_struct_ops_link, link);
1254 st_map = container_of(new_map, struct bpf_struct_ops_map, map);
1255
1256 if (!bpf_struct_ops_valid_to_reg(map: new_map))
1257 return -EINVAL;
1258
1259 if (!st_map->st_ops_desc->st_ops->update)
1260 return -EOPNOTSUPP;
1261
1262 mutex_lock(&update_mutex);
1263
1264 old_map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex));
1265 if (!old_map) {
1266 err = -ENOLINK;
1267 goto err_out;
1268 }
1269 if (expected_old_map && old_map != expected_old_map) {
1270 err = -EPERM;
1271 goto err_out;
1272 }
1273
1274 old_st_map = container_of(old_map, struct bpf_struct_ops_map, map);
1275 /* The new and old struct_ops must be the same type. */
1276 if (st_map->st_ops_desc != old_st_map->st_ops_desc) {
1277 err = -EINVAL;
1278 goto err_out;
1279 }
1280
1281 err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data, link);
1282 if (err)
1283 goto err_out;
1284
1285 bpf_map_inc(map: new_map);
1286 rcu_assign_pointer(st_link->map, new_map);
1287 bpf_map_put(map: old_map);
1288
1289err_out:
1290 mutex_unlock(lock: &update_mutex);
1291
1292 return err;
1293}
1294
1295static int bpf_struct_ops_map_link_detach(struct bpf_link *link)
1296{
1297 struct bpf_struct_ops_link *st_link = container_of(link, struct bpf_struct_ops_link, link);
1298 struct bpf_struct_ops_map *st_map;
1299 struct bpf_map *map;
1300
1301 mutex_lock(&update_mutex);
1302
1303 map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex));
1304 if (!map) {
1305 mutex_unlock(lock: &update_mutex);
1306 return 0;
1307 }
1308 st_map = container_of(map, struct bpf_struct_ops_map, map);
1309
1310 st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link);
1311
1312 RCU_INIT_POINTER(st_link->map, NULL);
1313 /* Pair with bpf_map_get() in bpf_struct_ops_link_create() or
1314 * bpf_map_inc() in bpf_struct_ops_map_link_update().
1315 */
1316 bpf_map_put(map: &st_map->map);
1317
1318 mutex_unlock(lock: &update_mutex);
1319
1320 wake_up_interruptible_poll(&st_link->wait_hup, EPOLLHUP);
1321
1322 return 0;
1323}
1324
1325static __poll_t bpf_struct_ops_map_link_poll(struct file *file,
1326 struct poll_table_struct *pts)
1327{
1328 struct bpf_struct_ops_link *st_link = file->private_data;
1329
1330 poll_wait(filp: file, wait_address: &st_link->wait_hup, p: pts);
1331
1332 return rcu_access_pointer(st_link->map) ? 0 : EPOLLHUP;
1333}
1334
1335static const struct bpf_link_ops bpf_struct_ops_map_lops = {
1336 .dealloc = bpf_struct_ops_map_link_dealloc,
1337 .detach = bpf_struct_ops_map_link_detach,
1338 .show_fdinfo = bpf_struct_ops_map_link_show_fdinfo,
1339 .fill_link_info = bpf_struct_ops_map_link_fill_link_info,
1340 .update_map = bpf_struct_ops_map_link_update,
1341 .poll = bpf_struct_ops_map_link_poll,
1342};
1343
1344int bpf_struct_ops_link_create(union bpf_attr *attr)
1345{
1346 struct bpf_struct_ops_link *link = NULL;
1347 struct bpf_link_primer link_primer;
1348 struct bpf_struct_ops_map *st_map;
1349 struct bpf_map *map;
1350 int err;
1351
1352 map = bpf_map_get(ufd: attr->link_create.map_fd);
1353 if (IS_ERR(ptr: map))
1354 return PTR_ERR(ptr: map);
1355
1356 st_map = (struct bpf_struct_ops_map *)map;
1357
1358 if (!bpf_struct_ops_valid_to_reg(map)) {
1359 err = -EINVAL;
1360 goto err_out;
1361 }
1362
1363 link = kzalloc(sizeof(*link), GFP_USER);
1364 if (!link) {
1365 err = -ENOMEM;
1366 goto err_out;
1367 }
1368 bpf_link_init(link: &link->link, type: BPF_LINK_TYPE_STRUCT_OPS, ops: &bpf_struct_ops_map_lops, NULL,
1369 attach_type: attr->link_create.attach_type);
1370
1371 err = bpf_link_prime(link: &link->link, primer: &link_primer);
1372 if (err)
1373 goto err_out;
1374
1375 init_waitqueue_head(&link->wait_hup);
1376
1377 /* Hold the update_mutex such that the subsystem cannot
1378 * do link->ops->detach() before the link is fully initialized.
1379 */
1380 mutex_lock(&update_mutex);
1381 err = st_map->st_ops_desc->st_ops->reg(st_map->kvalue.data, &link->link);
1382 if (err) {
1383 mutex_unlock(lock: &update_mutex);
1384 bpf_link_cleanup(primer: &link_primer);
1385 link = NULL;
1386 goto err_out;
1387 }
1388 RCU_INIT_POINTER(link->map, map);
1389 mutex_unlock(lock: &update_mutex);
1390
1391 return bpf_link_settle(primer: &link_primer);
1392
1393err_out:
1394 bpf_map_put(map);
1395 kfree(objp: link);
1396 return err;
1397}
1398
1399void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)
1400{
1401 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
1402
1403 info->btf_vmlinux_id = btf_obj_id(btf: st_map->btf);
1404}
1405

source code of linux/kernel/bpf/bpf_struct_ops.c