1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2020 NVIDIA Corporation */
3
4#include <linux/dma-fence-array.h>
5#include <linux/dma-mapping.h>
6#include <linux/file.h>
7#include <linux/host1x.h>
8#include <linux/iommu.h>
9#include <linux/kref.h>
10#include <linux/list.h>
11#include <linux/nospec.h>
12#include <linux/pm_runtime.h>
13#include <linux/scatterlist.h>
14#include <linux/slab.h>
15#include <linux/sync_file.h>
16
17#include <drm/drm_drv.h>
18#include <drm/drm_file.h>
19#include <drm/drm_syncobj.h>
20
21#include "drm.h"
22#include "gem.h"
23#include "submit.h"
24#include "uapi.h"
25
26#define SUBMIT_ERR(context, fmt, ...) \
27 dev_err_ratelimited(context->client->base.dev, \
28 "%s: job submission failed: " fmt "\n", \
29 current->comm, ##__VA_ARGS__)
30
31struct gather_bo {
32 struct host1x_bo base;
33
34 struct kref ref;
35
36 struct device *dev;
37 u32 *gather_data;
38 dma_addr_t gather_data_dma;
39 size_t gather_data_words;
40};
41
42static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo)
43{
44 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
45
46 kref_get(kref: &bo->ref);
47
48 return host_bo;
49}
50
51static void gather_bo_release(struct kref *ref)
52{
53 struct gather_bo *bo = container_of(ref, struct gather_bo, ref);
54
55 dma_free_attrs(dev: bo->dev, size: bo->gather_data_words * 4, cpu_addr: bo->gather_data, dma_handle: bo->gather_data_dma,
56 attrs: 0);
57 kfree(objp: bo);
58}
59
60static void gather_bo_put(struct host1x_bo *host_bo)
61{
62 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
63
64 kref_put(kref: &bo->ref, release: gather_bo_release);
65}
66
67static struct host1x_bo_mapping *
68gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction)
69{
70 struct gather_bo *gather = container_of(bo, struct gather_bo, base);
71 struct host1x_bo_mapping *map;
72 int err;
73
74 map = kzalloc(sizeof(*map), GFP_KERNEL);
75 if (!map)
76 return ERR_PTR(error: -ENOMEM);
77
78 kref_init(kref: &map->ref);
79 map->bo = host1x_bo_get(bo);
80 map->direction = direction;
81 map->dev = dev;
82
83 map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL);
84 if (!map->sgt) {
85 err = -ENOMEM;
86 goto free;
87 }
88
89 err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma,
90 gather->gather_data_words * 4);
91 if (err)
92 goto free_sgt;
93
94 err = dma_map_sgtable(dev, sgt: map->sgt, dir: direction, attrs: 0);
95 if (err)
96 goto free_sgt;
97
98 map->phys = sg_dma_address(map->sgt->sgl);
99 map->size = gather->gather_data_words * 4;
100 map->chunks = err;
101
102 return map;
103
104free_sgt:
105 sg_free_table(map->sgt);
106 kfree(objp: map->sgt);
107free:
108 kfree(objp: map);
109 return ERR_PTR(error: err);
110}
111
112static void gather_bo_unpin(struct host1x_bo_mapping *map)
113{
114 if (!map)
115 return;
116
117 dma_unmap_sgtable(dev: map->dev, sgt: map->sgt, dir: map->direction, attrs: 0);
118 sg_free_table(map->sgt);
119 kfree(objp: map->sgt);
120 host1x_bo_put(bo: map->bo);
121
122 kfree(objp: map);
123}
124
125static void *gather_bo_mmap(struct host1x_bo *host_bo)
126{
127 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
128
129 return bo->gather_data;
130}
131
132static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr)
133{
134}
135
136static const struct host1x_bo_ops gather_bo_ops = {
137 .get = gather_bo_get,
138 .put = gather_bo_put,
139 .pin = gather_bo_pin,
140 .unpin = gather_bo_unpin,
141 .mmap = gather_bo_mmap,
142 .munmap = gather_bo_munmap,
143};
144
145static struct tegra_drm_mapping *
146tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id)
147{
148 struct tegra_drm_mapping *mapping;
149
150 xa_lock(&context->mappings);
151
152 mapping = xa_load(&context->mappings, index: id);
153 if (mapping)
154 kref_get(kref: &mapping->ref);
155
156 xa_unlock(&context->mappings);
157
158 return mapping;
159}
160
161static void *alloc_copy_user_array(void __user *from, size_t count, size_t size)
162{
163 size_t copy_len;
164 void *data;
165
166 if (check_mul_overflow(count, size, &copy_len))
167 return ERR_PTR(error: -EINVAL);
168
169 if (copy_len > 0x4000)
170 return ERR_PTR(error: -E2BIG);
171
172 data = vmemdup_user(from, copy_len);
173 if (IS_ERR(ptr: data))
174 return ERR_CAST(ptr: data);
175
176 return data;
177}
178
179static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev,
180 struct tegra_drm_context *context,
181 struct drm_tegra_channel_submit *args)
182{
183 struct gather_bo *bo;
184 size_t copy_len;
185
186 if (args->gather_data_words == 0) {
187 SUBMIT_ERR(context, "gather_data_words cannot be zero");
188 return -EINVAL;
189 }
190
191 if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, &copy_len)) {
192 SUBMIT_ERR(context, "gather_data_words is too large");
193 return -EINVAL;
194 }
195
196 bo = kzalloc(sizeof(*bo), GFP_KERNEL);
197 if (!bo) {
198 SUBMIT_ERR(context, "failed to allocate memory for bo info");
199 return -ENOMEM;
200 }
201
202 host1x_bo_init(bo: &bo->base, ops: &gather_bo_ops);
203 kref_init(kref: &bo->ref);
204 bo->dev = dev;
205
206 bo->gather_data = dma_alloc_attrs(dev, size: copy_len, dma_handle: &bo->gather_data_dma,
207 GFP_KERNEL | __GFP_NOWARN, attrs: 0);
208 if (!bo->gather_data) {
209 SUBMIT_ERR(context, "failed to allocate memory for gather data");
210 kfree(objp: bo);
211 return -ENOMEM;
212 }
213
214 if (copy_from_user(to: bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), n: copy_len)) {
215 SUBMIT_ERR(context, "failed to copy gather data from userspace");
216 dma_free_attrs(dev, size: copy_len, cpu_addr: bo->gather_data, dma_handle: bo->gather_data_dma, attrs: 0);
217 kfree(objp: bo);
218 return -EFAULT;
219 }
220
221 bo->gather_data_words = args->gather_data_words;
222
223 *pbo = bo;
224
225 return 0;
226}
227
228static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo,
229 struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping)
230{
231 /* TODO check that target_offset is within bounds */
232 dma_addr_t iova = mapping->iova + buf->reloc.target_offset;
233 u32 written_ptr;
234
235#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
236 if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT)
237 iova |= BIT_ULL(39);
238#endif
239
240 written_ptr = iova >> buf->reloc.shift;
241
242 if (buf->reloc.gather_offset_words >= bo->gather_data_words) {
243 SUBMIT_ERR(context,
244 "relocation has too large gather offset (%u vs gather length %zu)",
245 buf->reloc.gather_offset_words, bo->gather_data_words);
246 return -EINVAL;
247 }
248
249 buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words,
250 bo->gather_data_words);
251
252 bo->gather_data[buf->reloc.gather_offset_words] = written_ptr;
253
254 return 0;
255}
256
257static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo,
258 struct drm_tegra_channel_submit *args,
259 struct tegra_drm_submit_data *job_data)
260{
261 struct tegra_drm_used_mapping *mappings;
262 struct drm_tegra_submit_buf *bufs;
263 int err;
264 u32 i;
265
266 bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), count: args->num_bufs,
267 size: sizeof(*bufs));
268 if (IS_ERR(ptr: bufs)) {
269 SUBMIT_ERR(context, "failed to copy bufs array from userspace");
270 return PTR_ERR(ptr: bufs);
271 }
272
273 mappings = kcalloc(args->num_bufs, sizeof(*mappings), GFP_KERNEL);
274 if (!mappings) {
275 SUBMIT_ERR(context, "failed to allocate memory for mapping info");
276 err = -ENOMEM;
277 goto done;
278 }
279
280 for (i = 0; i < args->num_bufs; i++) {
281 struct drm_tegra_submit_buf *buf = &bufs[i];
282 struct tegra_drm_mapping *mapping;
283
284 if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) {
285 SUBMIT_ERR(context, "invalid flag specified for buffer");
286 err = -EINVAL;
287 goto drop_refs;
288 }
289
290 mapping = tegra_drm_mapping_get(context, id: buf->mapping);
291 if (!mapping) {
292 SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping);
293 err = -EINVAL;
294 goto drop_refs;
295 }
296
297 err = submit_write_reloc(context, bo, buf, mapping);
298 if (err) {
299 tegra_drm_mapping_put(mapping);
300 goto drop_refs;
301 }
302
303 mappings[i].mapping = mapping;
304 mappings[i].flags = buf->flags;
305 }
306
307 job_data->used_mappings = mappings;
308 job_data->num_used_mappings = i;
309
310 err = 0;
311
312 goto done;
313
314drop_refs:
315 while (i--)
316 tegra_drm_mapping_put(mapping: mappings[i].mapping);
317
318 kfree(objp: mappings);
319 job_data->used_mappings = NULL;
320
321done:
322 kvfree(addr: bufs);
323
324 return err;
325}
326
327static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job,
328 struct xarray *syncpoints, struct drm_tegra_channel_submit *args)
329{
330 struct host1x_syncpt *sp;
331
332 if (args->syncpt.flags) {
333 SUBMIT_ERR(context, "invalid flag specified for syncpt");
334 return -EINVAL;
335 }
336
337 /* Syncpt ref will be dropped on job release */
338 sp = xa_load(syncpoints, index: args->syncpt.id);
339 if (!sp) {
340 SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated");
341 return -EINVAL;
342 }
343
344 job->syncpt = host1x_syncpt_get(sp);
345 job->syncpt_incrs = args->syncpt.increments;
346
347 return 0;
348}
349
350static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context,
351 struct drm_tegra_submit_cmd_gather_uptr *cmd,
352 struct gather_bo *bo, u32 *offset,
353 struct tegra_drm_submit_data *job_data,
354 u32 *class)
355{
356 u32 next_offset;
357
358 if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) {
359 SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command");
360 return -EINVAL;
361 }
362
363 /* Check for maximum gather size */
364 if (cmd->words > 16383) {
365 SUBMIT_ERR(context, "too many words in GATHER_UPTR command");
366 return -EINVAL;
367 }
368
369 if (check_add_overflow(*offset, cmd->words, &next_offset)) {
370 SUBMIT_ERR(context, "too many total words in job");
371 return -EINVAL;
372 }
373
374 if (next_offset > bo->gather_data_words) {
375 SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data");
376 return -EINVAL;
377 }
378
379 if (tegra_drm_fw_validate(client: context->client, data: bo->gather_data, start: *offset,
380 words: cmd->words, submit: job_data, job_class: class)) {
381 SUBMIT_ERR(context, "job was rejected by firewall");
382 return -EINVAL;
383 }
384
385 host1x_job_add_gather(job, bo: &bo->base, words: cmd->words, offset: *offset * 4);
386
387 *offset = next_offset;
388
389 return 0;
390}
391
392static struct host1x_job *
393submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo,
394 struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data,
395 struct xarray *syncpoints)
396{
397 struct drm_tegra_submit_cmd *cmds;
398 u32 i, gather_offset = 0, class;
399 struct host1x_job *job;
400 int err;
401
402 /* Set initial class for firewall. */
403 class = context->client->base.class;
404
405 cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), count: args->num_cmds,
406 size: sizeof(*cmds));
407 if (IS_ERR(ptr: cmds)) {
408 SUBMIT_ERR(context, "failed to copy cmds array from userspace");
409 return ERR_CAST(ptr: cmds);
410 }
411
412 job = host1x_job_alloc(ch: context->channel, num_cmdbufs: args->num_cmds, num_relocs: 0, skip_firewall: true);
413 if (!job) {
414 SUBMIT_ERR(context, "failed to allocate memory for job");
415 job = ERR_PTR(error: -ENOMEM);
416 goto done;
417 }
418
419 err = submit_get_syncpt(context, job, syncpoints, args);
420 if (err < 0)
421 goto free_job;
422
423 job->client = &context->client->base;
424 job->class = context->client->base.class;
425 job->serialize = true;
426
427 for (i = 0; i < args->num_cmds; i++) {
428 struct drm_tegra_submit_cmd *cmd = &cmds[i];
429
430 if (cmd->flags) {
431 SUBMIT_ERR(context, "unknown flags given for cmd");
432 err = -EINVAL;
433 goto free_job;
434 }
435
436 if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) {
437 err = submit_job_add_gather(job, context, cmd: &cmd->gather_uptr, bo,
438 offset: &gather_offset, job_data, class: &class);
439 if (err)
440 goto free_job;
441 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) {
442 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) {
443 SUBMIT_ERR(context, "non-zero reserved value");
444 err = -EINVAL;
445 goto free_job;
446 }
447
448 host1x_job_add_wait(job, id: cmd->wait_syncpt.id, thresh: cmd->wait_syncpt.value,
449 relative: false, next_class: class);
450 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) {
451 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) {
452 SUBMIT_ERR(context, "non-zero reserved value");
453 err = -EINVAL;
454 goto free_job;
455 }
456
457 if (cmd->wait_syncpt.id != args->syncpt.id) {
458 SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job");
459 err = -EINVAL;
460 goto free_job;
461 }
462
463 host1x_job_add_wait(job, id: cmd->wait_syncpt.id, thresh: cmd->wait_syncpt.value,
464 relative: true, next_class: class);
465 } else {
466 SUBMIT_ERR(context, "unknown cmd type");
467 err = -EINVAL;
468 goto free_job;
469 }
470 }
471
472 if (gather_offset == 0) {
473 SUBMIT_ERR(context, "job must have at least one gather");
474 err = -EINVAL;
475 goto free_job;
476 }
477
478 goto done;
479
480free_job:
481 host1x_job_put(job);
482 job = ERR_PTR(error: err);
483
484done:
485 kvfree(addr: cmds);
486
487 return job;
488}
489
490static void release_job(struct host1x_job *job)
491{
492 struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base);
493 struct tegra_drm_submit_data *job_data = job->user_data;
494 u32 i;
495
496 if (job->memory_context)
497 host1x_memory_context_put(cd: job->memory_context);
498
499 for (i = 0; i < job_data->num_used_mappings; i++)
500 tegra_drm_mapping_put(mapping: job_data->used_mappings[i].mapping);
501
502 kfree(objp: job_data->used_mappings);
503 kfree(objp: job_data);
504
505 pm_runtime_mark_last_busy(dev: client->base.dev);
506 pm_runtime_put_autosuspend(dev: client->base.dev);
507}
508
509int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data,
510 struct drm_file *file)
511{
512 struct tegra_drm_file *fpriv = file->driver_priv;
513 struct drm_tegra_channel_submit *args = data;
514 struct tegra_drm_submit_data *job_data;
515 struct drm_syncobj *syncobj = NULL;
516 struct tegra_drm_context *context;
517 struct host1x_job *job;
518 struct gather_bo *bo;
519 u32 i;
520 int err;
521
522 mutex_lock(&fpriv->lock);
523
524 context = xa_load(&fpriv->contexts, index: args->context);
525 if (!context) {
526 mutex_unlock(lock: &fpriv->lock);
527 pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__,
528 current->comm, args->context);
529 return -EINVAL;
530 }
531
532 if (args->syncobj_in) {
533 struct dma_fence *fence;
534
535 err = drm_syncobj_find_fence(file_private: file, handle: args->syncobj_in, point: 0, flags: 0, fence: &fence);
536 if (err) {
537 SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in);
538 goto unlock;
539 }
540
541 err = dma_fence_wait_timeout(fence, intr: true, timeout: msecs_to_jiffies(m: 10000));
542 dma_fence_put(fence);
543 if (err) {
544 SUBMIT_ERR(context, "wait for syncobj_in timed out");
545 goto unlock;
546 }
547 }
548
549 if (args->syncobj_out) {
550 syncobj = drm_syncobj_find(file_private: file, handle: args->syncobj_out);
551 if (!syncobj) {
552 SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out);
553 err = -ENOENT;
554 goto unlock;
555 }
556 }
557
558 /* Allocate gather BO and copy gather words in. */
559 err = submit_copy_gather_data(pbo: &bo, dev: drm->dev, context, args);
560 if (err)
561 goto unlock;
562
563 job_data = kzalloc(sizeof(*job_data), GFP_KERNEL);
564 if (!job_data) {
565 SUBMIT_ERR(context, "failed to allocate memory for job data");
566 err = -ENOMEM;
567 goto put_bo;
568 }
569
570 /* Get data buffer mappings and do relocation patching. */
571 err = submit_process_bufs(context, bo, args, job_data);
572 if (err)
573 goto free_job_data;
574
575 /* Allocate host1x_job and add gathers and waits to it. */
576 job = submit_create_job(context, bo, args, job_data, syncpoints: &fpriv->syncpoints);
577 if (IS_ERR(ptr: job)) {
578 err = PTR_ERR(ptr: job);
579 goto free_job_data;
580 }
581
582 /* Map gather data for Host1x. */
583 err = host1x_job_pin(job, dev: context->client->base.dev);
584 if (err) {
585 SUBMIT_ERR(context, "failed to pin job: %d", err);
586 goto put_job;
587 }
588
589 if (context->client->ops->get_streamid_offset) {
590 err = context->client->ops->get_streamid_offset(
591 context->client, &job->engine_streamid_offset);
592 if (err) {
593 SUBMIT_ERR(context, "failed to get streamid offset: %d", err);
594 goto unpin_job;
595 }
596 }
597
598 if (context->memory_context && context->client->ops->can_use_memory_ctx) {
599 bool supported;
600
601 err = context->client->ops->can_use_memory_ctx(context->client, &supported);
602 if (err) {
603 SUBMIT_ERR(context, "failed to detect if engine can use memory context: %d", err);
604 goto unpin_job;
605 }
606
607 if (supported) {
608 job->memory_context = context->memory_context;
609 host1x_memory_context_get(cd: job->memory_context);
610 }
611 } else if (context->client->ops->get_streamid_offset) {
612 /*
613 * Job submission will need to temporarily change stream ID,
614 * so need to tell it what to change it back to.
615 */
616 if (!tegra_dev_iommu_get_stream_id(dev: context->client->base.dev,
617 stream_id: &job->engine_fallback_streamid))
618 job->engine_fallback_streamid = TEGRA_STREAM_ID_BYPASS;
619 }
620
621 /* Boot engine. */
622 err = pm_runtime_resume_and_get(dev: context->client->base.dev);
623 if (err < 0) {
624 SUBMIT_ERR(context, "could not power up engine: %d", err);
625 goto put_memory_context;
626 }
627
628 job->user_data = job_data;
629 job->release = release_job;
630 job->timeout = 10000;
631
632 /*
633 * job_data is now part of job reference counting, so don't release
634 * it from here.
635 */
636 job_data = NULL;
637
638 /* Submit job to hardware. */
639 err = host1x_job_submit(job);
640 if (err) {
641 SUBMIT_ERR(context, "host1x job submission failed: %d", err);
642 goto unpin_job;
643 }
644
645 /* Return postfences to userspace and add fences to DMA reservations. */
646 args->syncpt.value = job->syncpt_end;
647
648 if (syncobj) {
649 struct dma_fence *fence = host1x_fence_create(sp: job->syncpt, threshold: job->syncpt_end, timeout: true);
650 if (IS_ERR(ptr: fence)) {
651 err = PTR_ERR(ptr: fence);
652 SUBMIT_ERR(context, "failed to create postfence: %d", err);
653 }
654
655 drm_syncobj_replace_fence(syncobj, fence);
656 }
657
658 goto put_job;
659
660put_memory_context:
661 if (job->memory_context)
662 host1x_memory_context_put(cd: job->memory_context);
663unpin_job:
664 host1x_job_unpin(job);
665put_job:
666 host1x_job_put(job);
667free_job_data:
668 if (job_data && job_data->used_mappings) {
669 for (i = 0; i < job_data->num_used_mappings; i++)
670 tegra_drm_mapping_put(mapping: job_data->used_mappings[i].mapping);
671
672 kfree(objp: job_data->used_mappings);
673 }
674
675 kfree(objp: job_data);
676put_bo:
677 gather_bo_put(host_bo: &bo->base);
678unlock:
679 if (syncobj)
680 drm_syncobj_put(obj: syncobj);
681
682 mutex_unlock(lock: &fpriv->lock);
683 return err;
684}
685

source code of linux/drivers/gpu/drm/tegra/submit.c