1/*
2 * Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <rdma/rdma_user_ioctl.h>
34#include <rdma/uverbs_ioctl.h>
35#include "rdma_core.h"
36#include "uverbs.h"
37
38struct bundle_alloc_head {
39 struct_group_tagged(bundle_alloc_head_hdr, hdr,
40 struct bundle_alloc_head *next;
41 );
42 u8 data[];
43};
44
45struct bundle_priv {
46 /* Must be first */
47 struct bundle_alloc_head_hdr alloc_head;
48 struct bundle_alloc_head *allocated_mem;
49 size_t internal_avail;
50 size_t internal_used;
51
52 struct radix_tree_root *radix;
53 const struct uverbs_api_ioctl_method *method_elm;
54 void __rcu **radix_slots;
55 unsigned long radix_slots_len;
56 u32 method_key;
57
58 struct ib_uverbs_attr __user *user_attrs;
59 struct ib_uverbs_attr *uattrs;
60
61 DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
62 DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
63 DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);
64
65 /*
66 * Must be last. bundle ends in a flex array which overlaps
67 * internal_buffer.
68 */
69 struct uverbs_attr_bundle_hdr bundle;
70 u64 internal_buffer[32];
71};
72
73/*
74 * Each method has an absolute minimum amount of memory it needs to allocate,
75 * precompute that amount and determine if the onstack memory can be used or
76 * if allocation is need.
77 */
78void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
79 unsigned int num_attrs)
80{
81 struct bundle_priv *pbundle;
82 struct uverbs_attr_bundle *bundle;
83 size_t bundle_size =
84 offsetof(struct bundle_priv, internal_buffer) +
85 sizeof(*bundle->attrs) * method_elm->key_bitmap_len +
86 sizeof(*pbundle->uattrs) * num_attrs;
87
88 method_elm->use_stack = bundle_size <= sizeof(*pbundle);
89 method_elm->bundle_size =
90 ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer));
91
92 /* Do not want order-2 allocations for this. */
93 WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);
94}
95
96/**
97 * _uverbs_alloc() - Quickly allocate memory for use with a bundle
98 * @bundle: The bundle
99 * @size: Number of bytes to allocate
100 * @flags: Allocator flags
101 *
102 * The bundle allocator is intended for allocations that are connected with
103 * processing the system call related to the bundle. The allocated memory is
104 * always freed once the system call completes, and cannot be freed any other
105 * way.
106 *
107 * This tries to use a small pool of pre-allocated memory for performance.
108 */
109__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
110 gfp_t flags)
111{
112 struct bundle_priv *pbundle =
113 container_of(&bundle->hdr, struct bundle_priv, bundle);
114 size_t new_used;
115 void *res;
116
117 if (check_add_overflow(size, pbundle->internal_used, &new_used))
118 return ERR_PTR(error: -EOVERFLOW);
119
120 if (new_used > pbundle->internal_avail) {
121 struct bundle_alloc_head *buf;
122
123 buf = kvmalloc(struct_size(buf, data, size), flags);
124 if (!buf)
125 return ERR_PTR(error: -ENOMEM);
126 buf->next = pbundle->allocated_mem;
127 pbundle->allocated_mem = buf;
128 return buf->data;
129 }
130
131 res = (void *)pbundle->internal_buffer + pbundle->internal_used;
132 pbundle->internal_used =
133 ALIGN(new_used, sizeof(*pbundle->internal_buffer));
134 if (want_init_on_alloc(flags))
135 memset(res, 0, size);
136 return res;
137}
138EXPORT_SYMBOL(_uverbs_alloc);
139
140static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
141 u16 len)
142{
143 if (uattr->len > sizeof_field(struct ib_uverbs_attr, data))
144 return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len,
145 len: uattr->len - len);
146
147 return !memchr_inv(p: (const void *)&uattr->data + len,
148 c: 0, size: uattr->len - len);
149}
150
151static int uverbs_set_output(const struct uverbs_attr_bundle *bundle,
152 const struct uverbs_attr *attr)
153{
154 struct bundle_priv *pbundle =
155 container_of(&bundle->hdr, struct bundle_priv, bundle);
156 u16 flags;
157
158 flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
159 UVERBS_ATTR_F_VALID_OUTPUT;
160 if (put_user(flags,
161 &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
162 return -EFAULT;
163 return 0;
164}
165
166static int uverbs_process_idrs_array(struct bundle_priv *pbundle,
167 const struct uverbs_api_attr *attr_uapi,
168 struct uverbs_objs_arr_attr *attr,
169 struct ib_uverbs_attr *uattr,
170 u32 attr_bkey)
171{
172 struct uverbs_attr_bundle *bundle =
173 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
174 const struct uverbs_attr_spec *spec = &attr_uapi->spec;
175 size_t array_len;
176 u32 *idr_vals;
177 int ret = 0;
178 size_t i;
179
180 if (uattr->attr_data.reserved)
181 return -EINVAL;
182
183 if (uattr->len % sizeof(u32))
184 return -EINVAL;
185
186 array_len = uattr->len / sizeof(u32);
187 if (array_len < spec->u2.objs_arr.min_len ||
188 array_len > spec->u2.objs_arr.max_len)
189 return -EINVAL;
190
191 attr->uobjects =
192 uverbs_alloc(bundle,
193 array_size(array_len, sizeof(*attr->uobjects)));
194 if (IS_ERR(ptr: attr->uobjects))
195 return PTR_ERR(ptr: attr->uobjects);
196
197 /*
198 * Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects
199 * to store idrs array and avoid additional memory allocation. The
200 * idrs array is offset to the end of the uobjects array so we will be
201 * able to read idr and replace with a pointer.
202 */
203 idr_vals = (u32 *)(attr->uobjects + array_len) - array_len;
204
205 if (uattr->len > sizeof(uattr->data)) {
206 ret = copy_from_user(to: idr_vals, u64_to_user_ptr(uattr->data),
207 n: uattr->len);
208 if (ret)
209 return -EFAULT;
210 } else {
211 memcpy(idr_vals, &uattr->data, uattr->len);
212 }
213
214 for (i = 0; i != array_len; i++) {
215 attr->uobjects[i] = uverbs_get_uobject_from_file(
216 object_id: spec->u2.objs_arr.obj_type, access: spec->u2.objs_arr.access,
217 id: idr_vals[i], attrs: bundle);
218 if (IS_ERR(ptr: attr->uobjects[i])) {
219 ret = PTR_ERR(ptr: attr->uobjects[i]);
220 break;
221 }
222 }
223
224 attr->len = i;
225 __set_bit(attr_bkey, pbundle->spec_finalize);
226 return ret;
227}
228
229static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,
230 struct uverbs_objs_arr_attr *attr,
231 bool commit,
232 struct uverbs_attr_bundle *attrs)
233{
234 const struct uverbs_attr_spec *spec = &attr_uapi->spec;
235 size_t i;
236
237 for (i = 0; i != attr->len; i++)
238 uverbs_finalize_object(uobj: attr->uobjects[i],
239 access: spec->u2.objs_arr.access, hw_obj_valid: false, commit,
240 attrs);
241}
242
243static int uverbs_process_attr(struct bundle_priv *pbundle,
244 const struct uverbs_api_attr *attr_uapi,
245 struct ib_uverbs_attr *uattr, u32 attr_bkey)
246{
247 const struct uverbs_attr_spec *spec = &attr_uapi->spec;
248 struct uverbs_attr_bundle *bundle =
249 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
250 struct uverbs_attr *e = &bundle->attrs[attr_bkey];
251 const struct uverbs_attr_spec *val_spec = spec;
252 struct uverbs_obj_attr *o_attr;
253
254 switch (spec->type) {
255 case UVERBS_ATTR_TYPE_ENUM_IN:
256 if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems)
257 return -EOPNOTSUPP;
258
259 if (uattr->attr_data.enum_data.reserved)
260 return -EINVAL;
261
262 val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id];
263
264 /* Currently we only support PTR_IN based enums */
265 if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN)
266 return -EOPNOTSUPP;
267
268 e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id;
269 fallthrough;
270 case UVERBS_ATTR_TYPE_PTR_IN:
271 /* Ensure that any data provided by userspace beyond the known
272 * struct is zero. Userspace that knows how to use some future
273 * longer struct will fail here if used with an old kernel and
274 * non-zero content, making ABI compat/discovery simpler.
275 */
276 if (uattr->len > val_spec->u.ptr.len &&
277 val_spec->zero_trailing &&
278 !uverbs_is_attr_cleared(uattr, len: val_spec->u.ptr.len))
279 return -EOPNOTSUPP;
280
281 fallthrough;
282 case UVERBS_ATTR_TYPE_PTR_OUT:
283 if (uattr->len < val_spec->u.ptr.min_len ||
284 (!val_spec->zero_trailing &&
285 uattr->len > val_spec->u.ptr.len))
286 return -EINVAL;
287
288 if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN &&
289 uattr->attr_data.reserved)
290 return -EINVAL;
291
292 e->ptr_attr.uattr_idx = uattr - pbundle->uattrs;
293 e->ptr_attr.len = uattr->len;
294
295 if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(attr: e)) {
296 void *p;
297
298 p = uverbs_alloc(bundle, size: uattr->len);
299 if (IS_ERR(ptr: p))
300 return PTR_ERR(ptr: p);
301
302 e->ptr_attr.ptr = p;
303
304 if (copy_from_user(to: p, u64_to_user_ptr(uattr->data),
305 n: uattr->len))
306 return -EFAULT;
307 } else {
308 e->ptr_attr.data = uattr->data;
309 }
310 break;
311
312 case UVERBS_ATTR_TYPE_IDR:
313 case UVERBS_ATTR_TYPE_FD:
314 if (uattr->attr_data.reserved)
315 return -EINVAL;
316
317 if (uattr->len != 0)
318 return -EINVAL;
319
320 o_attr = &e->obj_attr;
321 o_attr->attr_elm = attr_uapi;
322
323 /*
324 * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and
325 * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64
326 * here without caring about truncation as we know that the
327 * IDR implementation today rejects negative IDs
328 */
329 o_attr->uobject = uverbs_get_uobject_from_file(
330 object_id: spec->u.obj.obj_type, access: spec->u.obj.access,
331 id: uattr->data_s64, attrs: bundle);
332 if (IS_ERR(ptr: o_attr->uobject))
333 return PTR_ERR(ptr: o_attr->uobject);
334 __set_bit(attr_bkey, pbundle->uobj_finalize);
335
336 if (spec->u.obj.access == UVERBS_ACCESS_NEW) {
337 unsigned int uattr_idx = uattr - pbundle->uattrs;
338 s64 id = o_attr->uobject->id;
339
340 /* Copy the allocated id to the user-space */
341 if (put_user(id, &pbundle->user_attrs[uattr_idx].data))
342 return -EFAULT;
343 }
344
345 break;
346
347 case UVERBS_ATTR_TYPE_RAW_FD:
348 if (uattr->attr_data.reserved || uattr->len != 0 ||
349 uattr->data_s64 < INT_MIN || uattr->data_s64 > INT_MAX)
350 return -EINVAL;
351 /* _uverbs_get_const_signed() is the accessor */
352 e->ptr_attr.data = uattr->data_s64;
353 break;
354
355 case UVERBS_ATTR_TYPE_IDRS_ARRAY:
356 return uverbs_process_idrs_array(pbundle, attr_uapi,
357 attr: &e->objs_arr_attr, uattr,
358 attr_bkey);
359 default:
360 return -EOPNOTSUPP;
361 }
362
363 return 0;
364}
365
366/*
367 * We search the radix tree with the method prefix and now we want to fast
368 * search the suffix bits to get a particular attribute pointer. It is not
369 * totally clear to me if this breaks the radix tree encasulation or not, but
370 * it uses the iter data to determine if the method iter points at the same
371 * chunk that will store the attribute, if so it just derefs it directly. By
372 * construction in most kernel configs the method and attrs will all fit in a
373 * single radix chunk, so in most cases this will have no search. Other cases
374 * this falls back to a full search.
375 */
376static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle,
377 u32 attr_key)
378{
379 void __rcu **slot;
380
381 if (likely(attr_key < pbundle->radix_slots_len)) {
382 void *entry;
383
384 slot = pbundle->radix_slots + attr_key;
385 entry = rcu_dereference_raw(*slot);
386 if (likely(!radix_tree_is_internal_node(entry) && entry))
387 return slot;
388 }
389
390 return radix_tree_lookup_slot(pbundle->radix,
391 index: pbundle->method_key | attr_key);
392}
393
394static int uverbs_set_attr(struct bundle_priv *pbundle,
395 struct ib_uverbs_attr *uattr)
396{
397 u32 attr_key = uapi_key_attr(id: uattr->attr_id);
398 u32 attr_bkey = uapi_bkey_attr(attr_key);
399 const struct uverbs_api_attr *attr;
400 void __rcu **slot;
401 int ret;
402
403 slot = uapi_get_attr_for_method(pbundle, attr_key);
404 if (!slot) {
405 /*
406 * Kernel does not support the attribute but user-space says it
407 * is mandatory
408 */
409 if (uattr->flags & UVERBS_ATTR_F_MANDATORY)
410 return -EPROTONOSUPPORT;
411 return 0;
412 }
413 attr = rcu_dereference_protected(*slot, true);
414
415 /* Reject duplicate attributes from user-space */
416 if (test_bit(attr_bkey, pbundle->bundle.attr_present))
417 return -EINVAL;
418
419 ret = uverbs_process_attr(pbundle, attr_uapi: attr, uattr, attr_bkey);
420 if (ret)
421 return ret;
422
423 __set_bit(attr_bkey, pbundle->bundle.attr_present);
424
425 return 0;
426}
427
428static int ib_uverbs_run_method(struct bundle_priv *pbundle,
429 unsigned int num_attrs)
430{
431 int (*handler)(struct uverbs_attr_bundle *attrs);
432 struct uverbs_attr_bundle *bundle =
433 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
434 size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
435 unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
436 unsigned int i;
437 int ret;
438
439 /* See uverbs_disassociate_api() */
440 handler = srcu_dereference(
441 pbundle->method_elm->handler,
442 &pbundle->bundle.ufile->device->disassociate_srcu);
443 if (!handler)
444 return -EIO;
445
446 pbundle->uattrs = uverbs_alloc(bundle, size: uattrs_size);
447 if (IS_ERR(ptr: pbundle->uattrs))
448 return PTR_ERR(ptr: pbundle->uattrs);
449 if (copy_from_user(to: pbundle->uattrs, from: pbundle->user_attrs, n: uattrs_size))
450 return -EFAULT;
451
452 for (i = 0; i != num_attrs; i++) {
453 ret = uverbs_set_attr(pbundle, uattr: &pbundle->uattrs[i]);
454 if (unlikely(ret))
455 return ret;
456 }
457
458 /* User space did not provide all the mandatory attributes */
459 if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
460 pbundle->bundle.attr_present,
461 pbundle->method_elm->key_bitmap_len)))
462 return -EINVAL;
463
464 if (pbundle->method_elm->has_udata)
465 uverbs_fill_udata(bundle, udata: &pbundle->bundle.driver_udata,
466 attr_in: UVERBS_ATTR_UHW_IN, attr_out: UVERBS_ATTR_UHW_OUT);
467 else
468 pbundle->bundle.driver_udata = (struct ib_udata){};
469
470 if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {
471 struct uverbs_obj_attr *destroy_attr = &bundle->attrs[destroy_bkey].obj_attr;
472
473 ret = uobj_destroy(uobj: destroy_attr->uobject, attrs: bundle);
474 if (ret)
475 return ret;
476 __clear_bit(destroy_bkey, pbundle->uobj_finalize);
477
478 ret = handler(bundle);
479 uobj_put_destroy(uobj: destroy_attr->uobject);
480 } else {
481 ret = handler(bundle);
482 }
483
484 /*
485 * Until the drivers are revised to use the bundle directly we have to
486 * assume that the driver wrote to its UHW_OUT and flag userspace
487 * appropriately.
488 */
489 if (!ret && pbundle->method_elm->has_udata) {
490 const struct uverbs_attr *attr =
491 uverbs_attr_get(attrs_bundle: bundle, idx: UVERBS_ATTR_UHW_OUT);
492
493 if (!IS_ERR(ptr: attr))
494 ret = uverbs_set_output(bundle, attr);
495 }
496
497 /*
498 * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can
499 * not invoke the method because the request is not supported. No
500 * other cases should return this code.
501 */
502 if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT))
503 return -EINVAL;
504
505 return ret;
506}
507
508static void bundle_destroy(struct bundle_priv *pbundle, bool commit)
509{
510 unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
511 struct uverbs_attr_bundle *bundle =
512 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
513 struct bundle_alloc_head *memblock;
514 unsigned int i;
515
516 /* fast path for simple uobjects */
517 i = -1;
518 while ((i = find_next_bit(addr: pbundle->uobj_finalize, size: key_bitmap_len,
519 offset: i + 1)) < key_bitmap_len) {
520 struct uverbs_attr *attr = &bundle->attrs[i];
521
522 uverbs_finalize_object(
523 uobj: attr->obj_attr.uobject,
524 access: attr->obj_attr.attr_elm->spec.u.obj.access,
525 test_bit(i, pbundle->uobj_hw_obj_valid),
526 commit, attrs: bundle);
527 }
528
529 i = -1;
530 while ((i = find_next_bit(addr: pbundle->spec_finalize, size: key_bitmap_len,
531 offset: i + 1)) < key_bitmap_len) {
532 struct uverbs_attr *attr = &bundle->attrs[i];
533 const struct uverbs_api_attr *attr_uapi;
534 void __rcu **slot;
535
536 slot = uapi_get_attr_for_method(
537 pbundle,
538 attr_key: pbundle->method_key | uapi_bkey_to_key_attr(attr_bkey: i));
539 if (WARN_ON(!slot))
540 continue;
541
542 attr_uapi = rcu_dereference_protected(*slot, true);
543
544 if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
545 uverbs_free_idrs_array(attr_uapi, attr: &attr->objs_arr_attr,
546 commit, attrs: bundle);
547 }
548 }
549
550 for (memblock = pbundle->allocated_mem; memblock;) {
551 struct bundle_alloc_head *tmp = memblock;
552
553 memblock = memblock->next;
554 kvfree(addr: tmp);
555 }
556}
557
558static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
559 struct ib_uverbs_ioctl_hdr *hdr,
560 struct ib_uverbs_attr __user *user_attrs)
561{
562 const struct uverbs_api_ioctl_method *method_elm;
563 struct uverbs_api *uapi = ufile->device->uapi;
564 struct radix_tree_iter attrs_iter;
565 struct bundle_priv *pbundle;
566 struct bundle_priv onstack;
567 void __rcu **slot;
568 int ret;
569
570 if (unlikely(hdr->driver_id != uapi->driver_id))
571 return -EINVAL;
572
573 slot = radix_tree_iter_lookup(
574 root: &uapi->radix, iter: &attrs_iter,
575 index: uapi_key_obj(id: hdr->object_id) |
576 uapi_key_ioctl_method(id: hdr->method_id));
577 if (unlikely(!slot))
578 return -EPROTONOSUPPORT;
579 method_elm = rcu_dereference_protected(*slot, true);
580
581 if (!method_elm->use_stack) {
582 pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL);
583 if (!pbundle)
584 return -ENOMEM;
585 pbundle->internal_avail =
586 method_elm->bundle_size -
587 offsetof(struct bundle_priv, internal_buffer);
588 pbundle->alloc_head.next = NULL;
589 pbundle->allocated_mem = container_of(&pbundle->alloc_head,
590 struct bundle_alloc_head, hdr);
591 } else {
592 pbundle = &onstack;
593 pbundle->internal_avail = sizeof(pbundle->internal_buffer);
594 pbundle->allocated_mem = NULL;
595 }
596
597 /* Space for the pbundle->bundle.attrs flex array */
598 pbundle->method_elm = method_elm;
599 pbundle->method_key = attrs_iter.index;
600 pbundle->bundle.ufile = ufile;
601 pbundle->bundle.context = NULL; /* only valid if bundle has uobject */
602 pbundle->radix = &uapi->radix;
603 pbundle->radix_slots = slot;
604 pbundle->radix_slots_len = radix_tree_chunk_size(iter: &attrs_iter);
605 pbundle->user_attrs = user_attrs;
606
607 pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
608 sizeof(*container_of(&pbundle->bundle,
609 struct uverbs_attr_bundle, hdr)->attrs),
610 sizeof(*pbundle->internal_buffer));
611 memset(pbundle->bundle.attr_present, 0,
612 sizeof(pbundle->bundle.attr_present));
613 memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
614 memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize));
615 memset(pbundle->uobj_hw_obj_valid, 0,
616 sizeof(pbundle->uobj_hw_obj_valid));
617
618 ret = ib_uverbs_run_method(pbundle, num_attrs: hdr->num_attrs);
619 bundle_destroy(pbundle, commit: ret == 0);
620 return ret;
621}
622
623long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
624{
625 struct ib_uverbs_file *file = filp->private_data;
626 struct ib_uverbs_ioctl_hdr __user *user_hdr =
627 (struct ib_uverbs_ioctl_hdr __user *)arg;
628 struct ib_uverbs_ioctl_hdr hdr;
629 int srcu_key;
630 int err;
631
632 if (unlikely(cmd != RDMA_VERBS_IOCTL))
633 return -ENOIOCTLCMD;
634
635 err = copy_from_user(to: &hdr, from: user_hdr, n: sizeof(hdr));
636 if (err)
637 return -EFAULT;
638
639 if (hdr.length > PAGE_SIZE ||
640 hdr.length != struct_size(&hdr, attrs, hdr.num_attrs))
641 return -EINVAL;
642
643 if (hdr.reserved1 || hdr.reserved2)
644 return -EPROTONOSUPPORT;
645
646 srcu_key = srcu_read_lock(ssp: &file->device->disassociate_srcu);
647 err = ib_uverbs_cmd_verbs(ufile: file, hdr: &hdr, user_attrs: user_hdr->attrs);
648 srcu_read_unlock(ssp: &file->device->disassociate_srcu, idx: srcu_key);
649 return err;
650}
651
652int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
653 size_t idx, u64 allowed_bits)
654{
655 const struct uverbs_attr *attr;
656 u64 flags;
657
658 attr = uverbs_attr_get(attrs_bundle, idx);
659 /* Missing attribute means 0 flags */
660 if (IS_ERR(ptr: attr)) {
661 *to = 0;
662 return 0;
663 }
664
665 /*
666 * New userspace code should use 8 bytes to pass flags, but we
667 * transparently support old userspaces that were using 4 bytes as
668 * well.
669 */
670 if (attr->ptr_attr.len == 8)
671 flags = attr->ptr_attr.data;
672 else if (attr->ptr_attr.len == 4)
673 flags = *(u32 *)&attr->ptr_attr.data;
674 else
675 return -EINVAL;
676
677 if (flags & ~allowed_bits)
678 return -EINVAL;
679
680 *to = flags;
681 return 0;
682}
683EXPORT_SYMBOL(uverbs_get_flags64);
684
685int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
686 size_t idx, u64 allowed_bits)
687{
688 u64 flags;
689 int ret;
690
691 ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
692 if (ret)
693 return ret;
694
695 if (flags > U32_MAX)
696 return -EINVAL;
697 *to = flags;
698
699 return 0;
700}
701EXPORT_SYMBOL(uverbs_get_flags32);
702
703/*
704 * Fill a ib_udata struct (core or uhw) using the given attribute IDs.
705 * This is primarily used to convert the UVERBS_ATTR_UHW() into the
706 * ib_udata format used by the drivers.
707 */
708void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,
709 struct ib_udata *udata, unsigned int attr_in,
710 unsigned int attr_out)
711{
712 struct bundle_priv *pbundle =
713 container_of(&bundle->hdr, struct bundle_priv, bundle);
714 struct uverbs_attr_bundle *bundle_aux =
715 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
716 const struct uverbs_attr *in =
717 uverbs_attr_get(attrs_bundle: bundle_aux, idx: attr_in);
718 const struct uverbs_attr *out =
719 uverbs_attr_get(attrs_bundle: bundle_aux, idx: attr_out);
720
721 if (!IS_ERR(ptr: in)) {
722 udata->inlen = in->ptr_attr.len;
723 if (uverbs_attr_ptr_is_inline(attr: in))
724 udata->inbuf =
725 &pbundle->user_attrs[in->ptr_attr.uattr_idx]
726 .data;
727 else
728 udata->inbuf = u64_to_user_ptr(in->ptr_attr.data);
729 } else {
730 udata->inbuf = NULL;
731 udata->inlen = 0;
732 }
733
734 if (!IS_ERR(ptr: out)) {
735 udata->outbuf = u64_to_user_ptr(out->ptr_attr.data);
736 udata->outlen = out->ptr_attr.len;
737 } else {
738 udata->outbuf = NULL;
739 udata->outlen = 0;
740 }
741}
742
743int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
744 const void *from, size_t size)
745{
746 const struct uverbs_attr *attr = uverbs_attr_get(attrs_bundle: bundle, idx);
747 size_t min_size;
748
749 if (IS_ERR(ptr: attr))
750 return PTR_ERR(ptr: attr);
751
752 min_size = min_t(size_t, attr->ptr_attr.len, size);
753 if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, n: min_size))
754 return -EFAULT;
755
756 return uverbs_set_output(bundle, attr);
757}
758EXPORT_SYMBOL(uverbs_copy_to);
759
760
761/*
762 * This is only used if the caller has directly used copy_to_use to write the
763 * data. It signals to user space that the buffer is filled in.
764 */
765int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx)
766{
767 const struct uverbs_attr *attr = uverbs_attr_get(attrs_bundle: bundle, idx);
768
769 if (IS_ERR(ptr: attr))
770 return PTR_ERR(ptr: attr);
771
772 return uverbs_set_output(bundle, attr);
773}
774
775int _uverbs_get_const_signed(s64 *to,
776 const struct uverbs_attr_bundle *attrs_bundle,
777 size_t idx, s64 lower_bound, u64 upper_bound,
778 s64 *def_val)
779{
780 const struct uverbs_attr *attr;
781
782 attr = uverbs_attr_get(attrs_bundle, idx);
783 if (IS_ERR(ptr: attr)) {
784 if ((PTR_ERR(ptr: attr) != -ENOENT) || !def_val)
785 return PTR_ERR(ptr: attr);
786
787 *to = *def_val;
788 } else {
789 *to = attr->ptr_attr.data;
790 }
791
792 if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound))
793 return -EINVAL;
794
795 return 0;
796}
797EXPORT_SYMBOL(_uverbs_get_const_signed);
798
799int _uverbs_get_const_unsigned(u64 *to,
800 const struct uverbs_attr_bundle *attrs_bundle,
801 size_t idx, u64 upper_bound, u64 *def_val)
802{
803 const struct uverbs_attr *attr;
804
805 attr = uverbs_attr_get(attrs_bundle, idx);
806 if (IS_ERR(ptr: attr)) {
807 if ((PTR_ERR(ptr: attr) != -ENOENT) || !def_val)
808 return PTR_ERR(ptr: attr);
809
810 *to = *def_val;
811 } else {
812 *to = attr->ptr_attr.data;
813 }
814
815 if (*to > upper_bound)
816 return -EINVAL;
817
818 return 0;
819}
820EXPORT_SYMBOL(_uverbs_get_const_unsigned);
821
822int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
823 size_t idx, const void *from, size_t size)
824{
825 const struct uverbs_attr *attr = uverbs_attr_get(attrs_bundle: bundle, idx);
826
827 if (IS_ERR(ptr: attr))
828 return PTR_ERR(ptr: attr);
829
830 if (size < attr->ptr_attr.len) {
831 if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size,
832 n: attr->ptr_attr.len - size))
833 return -EFAULT;
834 }
835 return uverbs_copy_to(bundle, idx, from, size);
836}
837EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);
838
839/* Once called an abort will call through to the type's destroy_hw() */
840void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
841 u16 idx)
842{
843 struct bundle_priv *pbundle =
844 container_of(&bundle->hdr, struct bundle_priv, bundle);
845
846 __set_bit(uapi_bkey_attr(uapi_key_attr(idx)),
847 pbundle->uobj_hw_obj_valid);
848}
849EXPORT_SYMBOL(uverbs_finalize_uobj_create);
850

source code of linux/drivers/infiniband/core/uverbs_ioctl.c