1	// SPDX-License-Identifier: GPL-2.0
2	// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
3	// Copyright (c) 2018, Linaro Limited
4
5	#include <linux/completion.h>
6	#include <linux/device.h>
7	#include <linux/dma-buf.h>
8	#include <linux/dma-mapping.h>
9	#include <linux/dma-resv.h>
10	#include <linux/idr.h>
11	#include <linux/list.h>
12	#include <linux/miscdevice.h>
13	#include <linux/module.h>
14	#include <linux/of_address.h>
15	#include <linux/of.h>
16	#include <linux/platform_device.h>
17	#include <linux/sort.h>
18	#include <linux/of_platform.h>
19	#include <linux/rpmsg.h>
20	#include <linux/scatterlist.h>
21	#include <linux/slab.h>
22	#include <linux/firmware/qcom/qcom_scm.h>
23	#include <uapi/misc/fastrpc.h>
24	#include <linux/of_reserved_mem.h>
25
26	#define ADSP_DOMAIN_ID (0)
27	#define MDSP_DOMAIN_ID (1)
28	#define SDSP_DOMAIN_ID (2)
29	#define CDSP_DOMAIN_ID (3)
30	#define GDSP_DOMAIN_ID (4)
31	#define FASTRPC_MAX_SESSIONS 14
32	#define FASTRPC_MAX_VMIDS 16
33	#define FASTRPC_ALIGN 128
34	#define FASTRPC_MAX_FDLIST 16
35	#define FASTRPC_MAX_CRCLIST 64
36	#define FASTRPC_PHYS(p) ((p) & 0xffffffff)
37	#define FASTRPC_CTX_MAX (256)
38	#define FASTRPC_INIT_HANDLE 1
39	#define FASTRPC_DSP_UTILITIES_HANDLE 2
40	#define FASTRPC_CTXID_MASK (0xFF0)
41	#define INIT_FILELEN_MAX (2 * 1024 * 1024)
42	#define INIT_FILE_NAMELEN_MAX (128)
43	#define FASTRPC_DEVICE_NAME "fastrpc"
44
45	/* Add memory to static PD pool, protection thru XPU */
46	#define ADSP_MMAP_HEAP_ADDR 4
47	/* MAP static DMA buffer on DSP User PD */
48	#define ADSP_MMAP_DMA_BUFFER 6
49	/* Add memory to static PD pool protection thru hypervisor */
50	#define ADSP_MMAP_REMOTE_HEAP_ADDR 8
51	/* Add memory to userPD pool, for user heap */
52	#define ADSP_MMAP_ADD_PAGES 0x1000
53	/* Add memory to userPD pool, for LLC heap */
54	#define ADSP_MMAP_ADD_PAGES_LLC 0x3000,
55
56	#define DSP_UNSUPPORTED_API (0x80000414)
57	/* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */
58	#define FASTRPC_MAX_DSP_ATTRIBUTES (256)
59	#define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES)
60
61	/* Retrives number of input buffers from the scalars parameter */
62	#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
63
64	/* Retrives number of output buffers from the scalars parameter */
65	#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
66
67	/* Retrives number of input handles from the scalars parameter */
68	#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
69
70	/* Retrives number of output handles from the scalars parameter */
71	#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
72
73	#define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \
74	REMOTE_SCALARS_OUTBUFS(sc) + \
75	REMOTE_SCALARS_INHANDLES(sc)+ \
76	REMOTE_SCALARS_OUTHANDLES(sc))
77	#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \
78	(((attr & 0x07) << 29) | \
79	((method & 0x1f) << 24) | \
80	((in & 0xff) << 16) | \
81	((out & 0xff) << 8) | \
82	((oin & 0x0f) << 4) | \
83	(oout & 0x0f))
84
85	#define FASTRPC_SCALARS(method, in, out) \
86	FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
87
88	#define FASTRPC_CREATE_PROCESS_NARGS 6
89	#define FASTRPC_CREATE_STATIC_PROCESS_NARGS 3
90	/* Remote Method id table */
91	#define FASTRPC_RMID_INIT_ATTACH 0
92	#define FASTRPC_RMID_INIT_RELEASE 1
93	#define FASTRPC_RMID_INIT_MMAP 4
94	#define FASTRPC_RMID_INIT_MUNMAP 5
95	#define FASTRPC_RMID_INIT_CREATE 6
96	#define FASTRPC_RMID_INIT_CREATE_ATTR 7
97	#define FASTRPC_RMID_INIT_CREATE_STATIC 8
98	#define FASTRPC_RMID_INIT_MEM_MAP 10
99	#define FASTRPC_RMID_INIT_MEM_UNMAP 11
100
101	/* Protection Domain(PD) ids */
102	#define ROOT_PD (0)
103	#define USER_PD (1)
104	#define SENSORS_PD (2)
105
106	#define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)
107
108	struct fastrpc_phy_page {
109	u64 addr; /* physical address */
110	u64 size; /* size of contiguous region */
111	};
112
113	struct fastrpc_invoke_buf {
114	u32 num; /* number of contiguous regions */
115	u32 pgidx; /* index to start of contiguous region */
116	};
117
118	struct fastrpc_remote_dmahandle {
119	s32 fd; /* dma handle fd */
120	u32 offset; /* dma handle offset */
121	u32 len; /* dma handle length */
122	};
123
124	struct fastrpc_remote_buf {
125	u64 pv; /* buffer pointer */
126	u64 len; /* length of buffer */
127	};
128
129	union fastrpc_remote_arg {
130	struct fastrpc_remote_buf buf;
131	struct fastrpc_remote_dmahandle dma;
132	};
133
134	struct fastrpc_mmap_rsp_msg {
135	u64 vaddr;
136	};
137
138	struct fastrpc_mmap_req_msg {
139	s32 client_id;
140	u32 flags;
141	u64 vaddr;
142	s32 num;
143	};
144
145	struct fastrpc_mem_map_req_msg {
146	s32 client_id;
147	s32 fd;
148	s32 offset;
149	u32 flags;
150	u64 vaddrin;
151	s32 num;
152	s32 data_len;
153	};
154
155	struct fastrpc_munmap_req_msg {
156	s32 client_id;
157	u64 vaddr;
158	u64 size;
159	};
160
161	struct fastrpc_mem_unmap_req_msg {
162	s32 client_id;
163	s32 fd;
164	u64 vaddrin;
165	u64 len;
166	};
167
168	struct fastrpc_msg {
169	int client_id; /* process client id */
170	int tid; /* thread id */
171	u64 ctx; /* invoke caller context */
172	u32 handle; /* handle to invoke */
173	u32 sc; /* scalars structure describing the data */
174	u64 addr; /* physical address */
175	u64 size; /* size of contiguous region */
176	};
177
178	struct fastrpc_invoke_rsp {
179	u64 ctx; /* invoke caller context */
180	int retval; /* invoke return value */
181	};
182
183	struct fastrpc_buf_overlap {
184	u64 start;
185	u64 end;
186	int raix;
187	u64 mstart;
188	u64 mend;
189	u64 offset;
190	};
191
192	struct fastrpc_buf {
193	struct fastrpc_user *fl;
194	struct dma_buf *dmabuf;
195	struct device *dev;
196	void *virt;
197	u64 phys;
198	u64 size;
199	/* Lock for dma buf attachments */
200	struct mutex lock;
201	struct list_head attachments;
202	/* mmap support */
203	struct list_head node; /* list of user requested mmaps */
204	uintptr_t raddr;
205	};
206
207	struct fastrpc_dma_buf_attachment {
208	struct device *dev;
209	struct sg_table sgt;
210	struct list_head node;
211	};
212
213	struct fastrpc_map {
214	struct list_head node;
215	struct fastrpc_user *fl;
216	int fd;
217	struct dma_buf *buf;
218	struct sg_table *table;
219	struct dma_buf_attachment *attach;
220	u64 phys;
221	u64 size;
222	void *va;
223	u64 len;
224	u64 raddr;
225	u32 attr;
226	struct kref refcount;
227	};
228
229	struct fastrpc_invoke_ctx {
230	int nscalars;
231	int nbufs;
232	int retval;
233	int pid;
234	int client_id;
235	u32 sc;
236	u32 *crc;
237	u64 ctxid;
238	u64 msg_sz;
239	struct kref refcount;
240	struct list_head node; /* list of ctxs */
241	struct completion work;
242	struct work_struct put_work;
243	struct fastrpc_msg msg;
244	struct fastrpc_user *fl;
245	union fastrpc_remote_arg *rpra;
246	struct fastrpc_map **maps;
247	struct fastrpc_buf *buf;
248	struct fastrpc_invoke_args *args;
249	struct fastrpc_buf_overlap *olaps;
250	struct fastrpc_channel_ctx *cctx;
251	};
252
253	struct fastrpc_session_ctx {
254	struct device *dev;
255	int sid;
256	bool used;
257	bool valid;
258	};
259
260	struct fastrpc_channel_ctx {
261	int domain_id;
262	int sesscount;
263	int vmcount;
264	struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS];
265	struct rpmsg_device *rpdev;
266	struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
267	spinlock_t lock;
268	struct idr ctx_idr;
269	struct list_head users;
270	struct kref refcount;
271	/* Flag if dsp attributes are cached */
272	bool valid_attributes;
273	u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES];
274	struct fastrpc_device *secure_fdevice;
275	struct fastrpc_device *fdevice;
276	struct fastrpc_buf *remote_heap;
277	struct list_head invoke_interrupted_mmaps;
278	bool secure;
279	bool unsigned_support;
280	u64 dma_mask;
281	};
282
283	struct fastrpc_device {
284	struct fastrpc_channel_ctx *cctx;
285	struct miscdevice miscdev;
286	bool secure;
287	};
288
289	struct fastrpc_user {
290	struct list_head user;
291	struct list_head maps;
292	struct list_head pending;
293	struct list_head mmaps;
294
295	struct fastrpc_channel_ctx *cctx;
296	struct fastrpc_session_ctx *sctx;
297	struct fastrpc_buf *init_mem;
298
299	int client_id;
300	int pd;
301	bool is_secure_dev;
302	/* Lock for lists */
303	spinlock_t lock;
304	/* lock for allocations */
305	struct mutex mutex;
306	};
307
308	static void fastrpc_free_map(struct kref *ref)
309	{
310	struct fastrpc_map *map;
311
312	map = container_of(ref, struct fastrpc_map, refcount);
313
314	if (map->table) {
315	if (map->attr & FASTRPC_ATTR_SECUREMAP) {
316	struct qcom_scm_vmperm perm;
317	int vmid = map->fl->cctx->vmperms[0].vmid;
318	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid);
319	int err = 0;
320
321	perm.vmid = QCOM_SCM_VMID_HLOS;
322	perm.perm = QCOM_SCM_PERM_RWX;
323	err = qcom_scm_assign_mem(mem_addr: map->phys, mem_sz: map->len,
324	src: &src_perms, newvm: &perm, dest_cnt: 1);
325	if (err) {
326	dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n",
327	map->phys, map->len, err);
328	return;
329	}
330	}
331	dma_buf_unmap_attachment_unlocked(attach: map->attach, sg_table: map->table,
332	direction: DMA_BIDIRECTIONAL);
333	dma_buf_detach(dmabuf: map->buf, attach: map->attach);
334	dma_buf_put(dmabuf: map->buf);
335	}
336
337	if (map->fl) {
338	spin_lock(lock: &map->fl->lock);
339	list_del(entry: &map->node);
340	spin_unlock(lock: &map->fl->lock);
341	map->fl = NULL;
342	}
343
344	kfree(objp: map);
345	}
346
347	static void fastrpc_map_put(struct fastrpc_map *map)
348	{
349	if (map)
350	kref_put(kref: &map->refcount, release: fastrpc_free_map);
351	}
352
353	static int fastrpc_map_get(struct fastrpc_map *map)
354	{
355	if (!map)
356	return -ENOENT;
357
358	return kref_get_unless_zero(kref: &map->refcount) ? 0 : -ENOENT;
359	}
360
361
362	static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd,
363	struct fastrpc_map **ppmap)
364	{
365	struct fastrpc_map *map = NULL;
366	struct dma_buf *buf;
367	int ret = -ENOENT;
368
369	buf = dma_buf_get(fd);
370	if (IS_ERR(ptr: buf))
371	return PTR_ERR(ptr: buf);
372
373	spin_lock(lock: &fl->lock);
374	list_for_each_entry(map, &fl->maps, node) {
375	if (map->fd != fd || map->buf != buf)
376	continue;
377
378	*ppmap = map;
379	ret = 0;
380	break;
381	}
382	spin_unlock(lock: &fl->lock);
383
384	dma_buf_put(dmabuf: buf);
385
386	return ret;
387	}
388
389	static void fastrpc_buf_free(struct fastrpc_buf *buf)
390	{
391	dma_free_coherent(dev: buf->dev, size: buf->size, cpu_addr: buf->virt,
392	FASTRPC_PHYS(buf->phys));
393	kfree(objp: buf);
394	}
395
396	static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
397	u64 size, struct fastrpc_buf **obuf)
398	{
399	struct fastrpc_buf *buf;
400
401	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
402	if (!buf)
403	return -ENOMEM;
404
405	INIT_LIST_HEAD(list: &buf->attachments);
406	INIT_LIST_HEAD(list: &buf->node);
407	mutex_init(&buf->lock);
408
409	buf->fl = fl;
410	buf->virt = NULL;
411	buf->phys = 0;
412	buf->size = size;
413	buf->dev = dev;
414	buf->raddr = 0;
415
416	buf->virt = dma_alloc_coherent(dev, size: buf->size, dma_handle: (dma_addr_t *)&buf->phys,
417	GFP_KERNEL);
418	if (!buf->virt) {
419	mutex_destroy(lock: &buf->lock);
420	kfree(objp: buf);
421	return -ENOMEM;
422	}
423
424	*obuf = buf;
425
426	return 0;
427	}
428
429	static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
430	u64 size, struct fastrpc_buf **obuf)
431	{
432	int ret;
433	struct fastrpc_buf *buf;
434
435	ret = __fastrpc_buf_alloc(fl, dev, size, obuf);
436	if (ret)
437	return ret;
438
439	buf = *obuf;
440
441	if (fl->sctx && fl->sctx->sid)
442	buf->phys += ((u64)fl->sctx->sid << 32);
443
444	return 0;
445	}
446
447	static int fastrpc_remote_heap_alloc(struct fastrpc_user *fl, struct device *dev,
448	u64 size, struct fastrpc_buf **obuf)
449	{
450	struct device *rdev = &fl->cctx->rpdev->dev;
451
452	return __fastrpc_buf_alloc(fl, dev: rdev, size, obuf);
453	}
454
455	static void fastrpc_channel_ctx_free(struct kref *ref)
456	{
457	struct fastrpc_channel_ctx *cctx;
458
459	cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);
460
461	kfree(objp: cctx);
462	}
463
464	static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx)
465	{
466	kref_get(kref: &cctx->refcount);
467	}
468
469	static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)
470	{
471	kref_put(kref: &cctx->refcount, release: fastrpc_channel_ctx_free);
472	}
473
474	static void fastrpc_context_free(struct kref *ref)
475	{
476	struct fastrpc_invoke_ctx *ctx;
477	struct fastrpc_channel_ctx *cctx;
478	unsigned long flags;
479	int i;
480
481	ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
482	cctx = ctx->cctx;
483
484	for (i = 0; i < ctx->nbufs; i++)
485	fastrpc_map_put(map: ctx->maps[i]);
486
487	if (ctx->buf)
488	fastrpc_buf_free(buf: ctx->buf);
489
490	spin_lock_irqsave(&cctx->lock, flags);
491	idr_remove(&cctx->ctx_idr, id: ctx->ctxid >> 4);
492	spin_unlock_irqrestore(lock: &cctx->lock, flags);
493
494	kfree(objp: ctx->maps);
495	kfree(objp: ctx->olaps);
496	kfree(objp: ctx);
497
498	fastrpc_channel_ctx_put(cctx);
499	}
500
501	static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx)
502	{
503	kref_get(kref: &ctx->refcount);
504	}
505
506	static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)
507	{
508	kref_put(kref: &ctx->refcount, release: fastrpc_context_free);
509	}
510
511	static void fastrpc_context_put_wq(struct work_struct *work)
512	{
513	struct fastrpc_invoke_ctx *ctx =
514	container_of(work, struct fastrpc_invoke_ctx, put_work);
515
516	fastrpc_context_put(ctx);
517	}
518
519	#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
520	static int olaps_cmp(const void *a, const void *b)
521	{
522	struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a;
523	struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b;
524	/* sort with lowest starting buffer first */
525	int st = CMP(pa->start, pb->start);
526	/* sort with highest ending buffer first */
527	int ed = CMP(pb->end, pa->end);
528
529	return st == 0 ? ed : st;
530	}
531
532	static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)
533	{
534	u64 max_end = 0;
535	int i;
536
537	for (i = 0; i < ctx->nbufs; ++i) {
538	ctx->olaps[i].start = ctx->args[i].ptr;
539	ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;
540	ctx->olaps[i].raix = i;
541	}
542
543	sort(base: ctx->olaps, num: ctx->nbufs, size: sizeof(*ctx->olaps), cmp_func: olaps_cmp, NULL);
544
545	for (i = 0; i < ctx->nbufs; ++i) {
546	/* Falling inside previous range */
547	if (ctx->olaps[i].start < max_end) {
548	ctx->olaps[i].mstart = max_end;
549	ctx->olaps[i].mend = ctx->olaps[i].end;
550	ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
551
552	if (ctx->olaps[i].end > max_end) {
553	max_end = ctx->olaps[i].end;
554	} else {
555	ctx->olaps[i].mend = 0;
556	ctx->olaps[i].mstart = 0;
557	}
558
559	} else {
560	ctx->olaps[i].mend = ctx->olaps[i].end;
561	ctx->olaps[i].mstart = ctx->olaps[i].start;
562	ctx->olaps[i].offset = 0;
563	max_end = ctx->olaps[i].end;
564	}
565	}
566	}
567
568	static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
569	struct fastrpc_user *user, u32 kernel, u32 sc,
570	struct fastrpc_invoke_args *args)
571	{
572	struct fastrpc_channel_ctx *cctx = user->cctx;
573	struct fastrpc_invoke_ctx *ctx = NULL;
574	unsigned long flags;
575	int ret;
576
577	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
578	if (!ctx)
579	return ERR_PTR(error: -ENOMEM);
580
581	INIT_LIST_HEAD(list: &ctx->node);
582	ctx->fl = user;
583	ctx->nscalars = REMOTE_SCALARS_LENGTH(sc);
584	ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) +
585	REMOTE_SCALARS_OUTBUFS(sc);
586
587	if (ctx->nscalars) {
588	ctx->maps = kcalloc(ctx->nscalars,
589	sizeof(*ctx->maps), GFP_KERNEL);
590	if (!ctx->maps) {
591	kfree(objp: ctx);
592	return ERR_PTR(error: -ENOMEM);
593	}
594	ctx->olaps = kcalloc(ctx->nscalars,
595	sizeof(*ctx->olaps), GFP_KERNEL);
596	if (!ctx->olaps) {
597	kfree(objp: ctx->maps);
598	kfree(objp: ctx);
599	return ERR_PTR(error: -ENOMEM);
600	}
601	ctx->args = args;
602	fastrpc_get_buff_overlaps(ctx);
603	}
604
605	/* Released in fastrpc_context_put() */
606	fastrpc_channel_ctx_get(cctx);
607
608	ctx->sc = sc;
609	ctx->retval = -1;
610	ctx->pid = current->pid;
611	ctx->client_id = user->client_id;
612	ctx->cctx = cctx;
613	init_completion(x: &ctx->work);
614	INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
615
616	spin_lock(lock: &user->lock);
617	list_add_tail(new: &ctx->node, head: &user->pending);
618	spin_unlock(lock: &user->lock);
619
620	spin_lock_irqsave(&cctx->lock, flags);
621	ret = idr_alloc_cyclic(&cctx->ctx_idr, ptr: ctx, start: 1,
622	FASTRPC_CTX_MAX, GFP_ATOMIC);
623	if (ret < 0) {
624	spin_unlock_irqrestore(lock: &cctx->lock, flags);
625	goto err_idr;
626	}
627	ctx->ctxid = ret << 4;
628	spin_unlock_irqrestore(lock: &cctx->lock, flags);
629
630	kref_init(kref: &ctx->refcount);
631
632	return ctx;
633	err_idr:
634	spin_lock(lock: &user->lock);
635	list_del(entry: &ctx->node);
636	spin_unlock(lock: &user->lock);
637	fastrpc_channel_ctx_put(cctx);
638	kfree(objp: ctx->maps);
639	kfree(objp: ctx->olaps);
640	kfree(objp: ctx);
641
642	return ERR_PTR(error: ret);
643	}
644
645	static struct sg_table *
646	fastrpc_map_dma_buf(struct dma_buf_attachment *attachment,
647	enum dma_data_direction dir)
648	{
649	struct fastrpc_dma_buf_attachment *a = attachment->priv;
650	struct sg_table *table;
651	int ret;
652
653	table = &a->sgt;
654
655	ret = dma_map_sgtable(dev: attachment->dev, sgt: table, dir, attrs: 0);
656	if (ret)
657	table = ERR_PTR(error: ret);
658	return table;
659	}
660
661	static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach,
662	struct sg_table *table,
663	enum dma_data_direction dir)
664	{
665	dma_unmap_sgtable(dev: attach->dev, sgt: table, dir, attrs: 0);
666	}
667
668	static void fastrpc_release(struct dma_buf *dmabuf)
669	{
670	struct fastrpc_buf *buffer = dmabuf->priv;
671
672	fastrpc_buf_free(buf: buffer);
673	}
674
675	static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf,
676	struct dma_buf_attachment *attachment)
677	{
678	struct fastrpc_dma_buf_attachment *a;
679	struct fastrpc_buf *buffer = dmabuf->priv;
680	int ret;
681
682	a = kzalloc(sizeof(*a), GFP_KERNEL);
683	if (!a)
684	return -ENOMEM;
685
686	ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt,
687	FASTRPC_PHYS(buffer->phys), buffer->size);
688	if (ret < 0) {
689	dev_err(buffer->dev, "failed to get scatterlist from DMA API\n");
690	kfree(objp: a);
691	return -EINVAL;
692	}
693
694	a->dev = attachment->dev;
695	INIT_LIST_HEAD(list: &a->node);
696	attachment->priv = a;
697
698	mutex_lock(&buffer->lock);
699	list_add(new: &a->node, head: &buffer->attachments);
700	mutex_unlock(lock: &buffer->lock);
701
702	return 0;
703	}
704
705	static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf,
706	struct dma_buf_attachment *attachment)
707	{
708	struct fastrpc_dma_buf_attachment *a = attachment->priv;
709	struct fastrpc_buf *buffer = dmabuf->priv;
710
711	mutex_lock(&buffer->lock);
712	list_del(entry: &a->node);
713	mutex_unlock(lock: &buffer->lock);
714	sg_free_table(&a->sgt);
715	kfree(objp: a);
716	}
717
718	static int fastrpc_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
719	{
720	struct fastrpc_buf *buf = dmabuf->priv;
721
722	iosys_map_set_vaddr(map, vaddr: buf->virt);
723
724	return 0;
725	}
726
727	static int fastrpc_mmap(struct dma_buf *dmabuf,
728	struct vm_area_struct *vma)
729	{
730	struct fastrpc_buf *buf = dmabuf->priv;
731	size_t size = vma->vm_end - vma->vm_start;
732
733	dma_resv_assert_held(dmabuf->resv);
734
735	return dma_mmap_coherent(buf->dev, vma, buf->virt,
736	FASTRPC_PHYS(buf->phys), size);
737	}
738
739	static const struct dma_buf_ops fastrpc_dma_buf_ops = {
740	.attach = fastrpc_dma_buf_attach,
741	.detach = fastrpc_dma_buf_detatch,
742	.map_dma_buf = fastrpc_map_dma_buf,
743	.unmap_dma_buf = fastrpc_unmap_dma_buf,
744	.mmap = fastrpc_mmap,
745	.vmap = fastrpc_vmap,
746	.release = fastrpc_release,
747	};
748
749	static int fastrpc_map_attach(struct fastrpc_user *fl, int fd,
750	u64 len, u32 attr, struct fastrpc_map **ppmap)
751	{
752	struct fastrpc_session_ctx *sess = fl->sctx;
753	struct fastrpc_map *map = NULL;
754	struct sg_table *table;
755	struct scatterlist *sgl = NULL;
756	int err = 0, sgl_index = 0;
757
758	map = kzalloc(sizeof(*map), GFP_KERNEL);
759	if (!map)
760	return -ENOMEM;
761
762	INIT_LIST_HEAD(list: &map->node);
763	kref_init(kref: &map->refcount);
764
765	map->fl = fl;
766	map->fd = fd;
767	map->buf = dma_buf_get(fd);
768	if (IS_ERR(ptr: map->buf)) {
769	err = PTR_ERR(ptr: map->buf);
770	goto get_err;
771	}
772
773	map->attach = dma_buf_attach(dmabuf: map->buf, dev: sess->dev);
774	if (IS_ERR(ptr: map->attach)) {
775	dev_err(sess->dev, "Failed to attach dmabuf\n");
776	err = PTR_ERR(ptr: map->attach);
777	goto attach_err;
778	}
779
780	table = dma_buf_map_attachment_unlocked(attach: map->attach, direction: DMA_BIDIRECTIONAL);
781	if (IS_ERR(ptr: table)) {
782	err = PTR_ERR(ptr: table);
783	goto map_err;
784	}
785	map->table = table;
786
787	if (attr & FASTRPC_ATTR_SECUREMAP) {
788	map->phys = sg_phys(sg: map->table->sgl);
789	} else {
790	map->phys = sg_dma_address(map->table->sgl);
791	map->phys += ((u64)fl->sctx->sid << 32);
792	}
793	for_each_sg(map->table->sgl, sgl, map->table->nents,
794	sgl_index)
795	map->size += sg_dma_len(sgl);
796	if (len > map->size) {
797	dev_dbg(sess->dev, "Bad size passed len 0x%llx map size 0x%llx\n",
798	len, map->size);
799	err = -EINVAL;
800	goto map_err;
801	}
802	map->va = sg_virt(sg: map->table->sgl);
803	map->len = len;
804
805	if (attr & FASTRPC_ATTR_SECUREMAP) {
806	/*
807	* If subsystem VMIDs are defined in DTSI, then do
808	* hyp_assign from HLOS to those VM(s)
809	*/
810	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
811	struct qcom_scm_vmperm dst_perms[2] = {0};
812
813	dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
814	dst_perms[0].perm = QCOM_SCM_PERM_RW;
815	dst_perms[1].vmid = fl->cctx->vmperms[0].vmid;
816	dst_perms[1].perm = QCOM_SCM_PERM_RWX;
817	map->attr = attr;
818	err = qcom_scm_assign_mem(mem_addr: map->phys, mem_sz: (u64)map->len, src: &src_perms, newvm: dst_perms, dest_cnt: 2);
819	if (err) {
820	dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n",
821	map->phys, map->len, err);
822	goto map_err;
823	}
824	}
825	spin_lock(lock: &fl->lock);
826	list_add_tail(new: &map->node, head: &fl->maps);
827	spin_unlock(lock: &fl->lock);
828	*ppmap = map;
829
830	return 0;
831
832	map_err:
833	dma_buf_detach(dmabuf: map->buf, attach: map->attach);
834	attach_err:
835	dma_buf_put(dmabuf: map->buf);
836	get_err:
837	fastrpc_map_put(map);
838
839	return err;
840	}
841
842	static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
843	u64 len, u32 attr, struct fastrpc_map **ppmap)
844	{
845	struct fastrpc_session_ctx *sess = fl->sctx;
846	int err = 0;
847
848	if (!fastrpc_map_lookup(fl, fd, ppmap)) {
849	if (!fastrpc_map_get(map: *ppmap))
850	return 0;
851	dev_dbg(sess->dev, "%s: Failed to get map fd=%d\n",
852	__func__, fd);
853	}
854
855	err = fastrpc_map_attach(fl, fd, len, attr, ppmap);
856
857	return err;
858	}
859
860	/*
861	* Fastrpc payload buffer with metadata looks like:
862	*
863	* >>>>>> START of METADATA <<<<<<<<<
864	* +---------------------------------+
865	* | Arguments |
866	* | type:(union fastrpc_remote_arg)|
867	* | (0 - N) |
868	* +---------------------------------+
869	* | Invoke Buffer list |
870	* | type:(struct fastrpc_invoke_buf)|
871	* | (0 - N) |
872	* +---------------------------------+
873	* | Page info list |
874	* | type:(struct fastrpc_phy_page) |
875	* | (0 - N) |
876	* +---------------------------------+
877	* | Optional info |
878	* |(can be specific to SoC/Firmware)|
879	* +---------------------------------+
880	* >>>>>>>> END of METADATA <<<<<<<<<
881	* +---------------------------------+
882	* | Inline ARGS |
883	* | (0-N) |
884	* +---------------------------------+
885	*/
886
887	static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)
888	{
889	int size = 0;
890
891	size = (sizeof(struct fastrpc_remote_buf) +
892	sizeof(struct fastrpc_invoke_buf) +
893	sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
894	sizeof(u64) * FASTRPC_MAX_FDLIST +
895	sizeof(u32) * FASTRPC_MAX_CRCLIST;
896
897	return size;
898	}
899
900	static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
901	{
902	u64 size = 0;
903	int oix;
904
905	size = ALIGN(metalen, FASTRPC_ALIGN);
906	for (oix = 0; oix < ctx->nbufs; oix++) {
907	int i = ctx->olaps[oix].raix;
908
909	if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
910
911	if (ctx->olaps[oix].offset == 0)
912	size = ALIGN(size, FASTRPC_ALIGN);
913
914	size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart);
915	}
916	}
917
918	return size;
919	}
920
921	static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
922	{
923	struct device *dev = ctx->fl->sctx->dev;
924	int i, err;
925
926	for (i = 0; i < ctx->nscalars; ++i) {
927
928	if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||
929	ctx->args[i].length == 0)
930	continue;
931
932	if (i < ctx->nbufs)
933	err = fastrpc_map_create(fl: ctx->fl, fd: ctx->args[i].fd,
934	len: ctx->args[i].length, attr: ctx->args[i].attr, ppmap: &ctx->maps[i]);
935	else
936	err = fastrpc_map_attach(fl: ctx->fl, fd: ctx->args[i].fd,
937	len: ctx->args[i].length, attr: ctx->args[i].attr, ppmap: &ctx->maps[i]);
938	if (err) {
939	dev_err(dev, "Error Creating map %d\n", err);
940	return -EINVAL;
941	}
942
943	}
944	return 0;
945	}
946
947	static struct fastrpc_invoke_buf *fastrpc_invoke_buf_start(union fastrpc_remote_arg *pra, int len)
948	{
949	return (struct fastrpc_invoke_buf *)(&pra[len]);
950	}
951
952	static struct fastrpc_phy_page *fastrpc_phy_page_start(struct fastrpc_invoke_buf *buf, int len)
953	{
954	return (struct fastrpc_phy_page *)(&buf[len]);
955	}
956
957	static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
958	{
959	struct device *dev = ctx->fl->sctx->dev;
960	union fastrpc_remote_arg *rpra;
961	struct fastrpc_invoke_buf *list;
962	struct fastrpc_phy_page *pages;
963	int inbufs, i, oix, err = 0;
964	u64 len, rlen, pkt_size;
965	u64 pg_start, pg_end;
966	uintptr_t args;
967	int metalen;
968
969	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
970	metalen = fastrpc_get_meta_size(ctx);
971	pkt_size = fastrpc_get_payload_size(ctx, metalen);
972
973	err = fastrpc_create_maps(ctx);
974	if (err)
975	return err;
976
977	ctx->msg_sz = pkt_size;
978
979	if (ctx->fl->sctx->sid)
980	err = fastrpc_buf_alloc(fl: ctx->fl, dev, size: pkt_size, obuf: &ctx->buf);
981	else
982	err = fastrpc_remote_heap_alloc(fl: ctx->fl, dev, size: pkt_size, obuf: &ctx->buf);
983	if (err)
984	return err;
985
986	memset(ctx->buf->virt, 0, pkt_size);
987	rpra = ctx->buf->virt;
988	list = fastrpc_invoke_buf_start(pra: rpra, len: ctx->nscalars);
989	pages = fastrpc_phy_page_start(buf: list, len: ctx->nscalars);
990	args = (uintptr_t)ctx->buf->virt + metalen;
991	rlen = pkt_size - metalen;
992	ctx->rpra = rpra;
993
994	for (oix = 0; oix < ctx->nbufs; ++oix) {
995	int mlen;
996
997	i = ctx->olaps[oix].raix;
998	len = ctx->args[i].length;
999
1000	rpra[i].buf.pv = 0;
1001	rpra[i].buf.len = len;
1002	list[i].num = len ? 1 : 0;
1003	list[i].pgidx = i;
1004
1005	if (!len)
1006	continue;
1007
1008	if (ctx->maps[i]) {
1009	struct vm_area_struct *vma = NULL;
1010
1011	rpra[i].buf.pv = (u64) ctx->args[i].ptr;
1012	pages[i].addr = ctx->maps[i]->phys;
1013
1014	mmap_read_lock(current->mm);
1015	vma = find_vma(current->mm, addr: ctx->args[i].ptr);
1016	if (vma)
1017	pages[i].addr += (ctx->args[i].ptr & PAGE_MASK) -
1018	vma->vm_start;
1019	mmap_read_unlock(current->mm);
1020
1021	pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;
1022	pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>
1023	PAGE_SHIFT;
1024	pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1025
1026	} else {
1027
1028	if (ctx->olaps[oix].offset == 0) {
1029	rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
1030	args = ALIGN(args, FASTRPC_ALIGN);
1031	}
1032
1033	mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
1034
1035	if (rlen < mlen)
1036	goto bail;
1037
1038	rpra[i].buf.pv = args - ctx->olaps[oix].offset;
1039	pages[i].addr = ctx->buf->phys -
1040	ctx->olaps[oix].offset +
1041	(pkt_size - rlen);
1042	pages[i].addr = pages[i].addr & PAGE_MASK;
1043
1044	pg_start = (rpra[i].buf.pv & PAGE_MASK) >> PAGE_SHIFT;
1045	pg_end = ((rpra[i].buf.pv + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
1046	pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1047	args = args + mlen;
1048	rlen -= mlen;
1049	}
1050
1051	if (i < inbufs && !ctx->maps[i]) {
1052	void *dst = (void *)(uintptr_t)rpra[i].buf.pv;
1053	void *src = (void *)(uintptr_t)ctx->args[i].ptr;
1054
1055	if (!kernel) {
1056	if (copy_from_user(to: dst, from: (void __user *)src,
1057	n: len)) {
1058	err = -EFAULT;
1059	goto bail;
1060	}
1061	} else {
1062	memcpy(dst, src, len);
1063	}
1064	}
1065	}
1066
1067	for (i = ctx->nbufs; i < ctx->nscalars; ++i) {
1068	list[i].num = ctx->args[i].length ? 1 : 0;
1069	list[i].pgidx = i;
1070	if (ctx->maps[i]) {
1071	pages[i].addr = ctx->maps[i]->phys;
1072	pages[i].size = ctx->maps[i]->size;
1073	}
1074	rpra[i].dma.fd = ctx->args[i].fd;
1075	rpra[i].dma.len = ctx->args[i].length;
1076	rpra[i].dma.offset = (u64) ctx->args[i].ptr;
1077	}
1078
1079	bail:
1080	if (err)
1081	dev_err(dev, "Error: get invoke args failed:%d\n", err);
1082
1083	return err;
1084	}
1085
1086	static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx,
1087	u32 kernel)
1088	{
1089	union fastrpc_remote_arg *rpra = ctx->rpra;
1090	struct fastrpc_user *fl = ctx->fl;
1091	struct fastrpc_map *mmap = NULL;
1092	struct fastrpc_invoke_buf *list;
1093	struct fastrpc_phy_page *pages;
1094	u64 *fdlist;
1095	int i, inbufs, outbufs, handles;
1096	int ret = 0;
1097
1098	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
1099	outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);
1100	handles = REMOTE_SCALARS_INHANDLES(ctx->sc) + REMOTE_SCALARS_OUTHANDLES(ctx->sc);
1101	list = fastrpc_invoke_buf_start(pra: rpra, len: ctx->nscalars);
1102	pages = fastrpc_phy_page_start(buf: list, len: ctx->nscalars);
1103	fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
1104
1105	for (i = inbufs; i < ctx->nbufs; ++i) {
1106	if (!ctx->maps[i]) {
1107	void *src = (void *)(uintptr_t)rpra[i].buf.pv;
1108	void *dst = (void *)(uintptr_t)ctx->args[i].ptr;
1109	u64 len = rpra[i].buf.len;
1110
1111	if (!kernel) {
1112	if (copy_to_user(to: (void __user *)dst, from: src, n: len)) {
1113	ret = -EFAULT;
1114	goto cleanup_fdlist;
1115	}
1116	} else {
1117	memcpy(dst, src, len);
1118	}
1119	}
1120	}
1121
1122	cleanup_fdlist:
1123	/* Clean up fdlist which is updated by DSP */
1124	for (i = 0; i < FASTRPC_MAX_FDLIST; i++) {
1125	if (!fdlist[i])
1126	break;
1127	if (!fastrpc_map_lookup(fl, fd: (int)fdlist[i], ppmap: &mmap))
1128	fastrpc_map_put(map: mmap);
1129	}
1130
1131	return ret;
1132	}
1133
1134	static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
1135	struct fastrpc_invoke_ctx *ctx,
1136	u32 kernel, uint32_t handle)
1137	{
1138	struct fastrpc_channel_ctx *cctx;
1139	struct fastrpc_user *fl = ctx->fl;
1140	struct fastrpc_msg *msg = &ctx->msg;
1141	int ret;
1142
1143	cctx = fl->cctx;
1144	msg->client_id = fl->client_id;
1145	msg->tid = current->pid;
1146
1147	if (kernel)
1148	msg->client_id = 0;
1149
1150	msg->ctx = ctx->ctxid | fl->pd;
1151	msg->handle = handle;
1152	msg->sc = ctx->sc;
1153	msg->addr = ctx->buf ? ctx->buf->phys : 0;
1154	msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
1155	fastrpc_context_get(ctx);
1156
1157	ret = rpmsg_send(ept: cctx->rpdev->ept, data: (void *)msg, len: sizeof(*msg));
1158
1159	if (ret)
1160	fastrpc_context_put(ctx);
1161
1162	return ret;
1163
1164	}
1165
1166	static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
1167	u32 handle, u32 sc,
1168	struct fastrpc_invoke_args *args)
1169	{
1170	struct fastrpc_invoke_ctx *ctx = NULL;
1171	struct fastrpc_buf *buf, *b;
1172
1173	int err = 0;
1174
1175	if (!fl->sctx)
1176	return -EINVAL;
1177
1178	if (!fl->cctx->rpdev)
1179	return -EPIPE;
1180
1181	if (handle == FASTRPC_INIT_HANDLE && !kernel) {
1182	dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n", handle);
1183	return -EPERM;
1184	}
1185
1186	ctx = fastrpc_context_alloc(user: fl, kernel, sc, args);
1187	if (IS_ERR(ptr: ctx))
1188	return PTR_ERR(ptr: ctx);
1189
1190	err = fastrpc_get_args(kernel, ctx);
1191	if (err)
1192	goto bail;
1193
1194	/* make sure that all CPU memory writes are seen by DSP */
1195	dma_wmb();
1196	/* Send invoke buffer to remote dsp */
1197	err = fastrpc_invoke_send(sctx: fl->sctx, ctx, kernel, handle);
1198	if (err)
1199	goto bail;
1200
1201	if (kernel) {
1202	if (!wait_for_completion_timeout(x: &ctx->work, timeout: 10 * HZ))
1203	err = -ETIMEDOUT;
1204	} else {
1205	err = wait_for_completion_interruptible(x: &ctx->work);
1206	}
1207
1208	if (err)
1209	goto bail;
1210
1211	/* make sure that all memory writes by DSP are seen by CPU */
1212	dma_rmb();
1213	/* populate all the output buffers with results */
1214	err = fastrpc_put_args(ctx, kernel);
1215	if (err)
1216	goto bail;
1217
1218	/* Check the response from remote dsp */
1219	err = ctx->retval;
1220	if (err)
1221	goto bail;
1222
1223	bail:
1224	if (err != -ERESTARTSYS && err != -ETIMEDOUT) {
1225	/* We are done with this compute context */
1226	spin_lock(lock: &fl->lock);
1227	list_del(entry: &ctx->node);
1228	spin_unlock(lock: &fl->lock);
1229	fastrpc_context_put(ctx);
1230	}
1231
1232	if (err == -ERESTARTSYS) {
1233	list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1234	list_del(entry: &buf->node);
1235	list_add_tail(new: &buf->node, head: &fl->cctx->invoke_interrupted_mmaps);
1236	}
1237	}
1238
1239	if (err)
1240	dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);
1241
1242	return err;
1243	}
1244
1245	static bool is_session_rejected(struct fastrpc_user *fl, bool unsigned_pd_request)
1246	{
1247	/* Check if the device node is non-secure and channel is secure*/
1248	if (!fl->is_secure_dev && fl->cctx->secure) {
1249	/*
1250	* Allow untrusted applications to offload only to Unsigned PD when
1251	* channel is configured as secure and block untrusted apps on channel
1252	* that does not support unsigned PD offload
1253	*/
1254	if (!fl->cctx->unsigned_support || !unsigned_pd_request) {
1255	dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD\n");
1256	return true;
1257	}
1258	}
1259
1260	return false;
1261	}
1262
1263	static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
1264	char __user *argp)
1265	{
1266	struct fastrpc_init_create_static init;
1267	struct fastrpc_invoke_args *args;
1268	struct fastrpc_phy_page pages[1];
1269	char *name;
1270	int err;
1271	bool scm_done = false;
1272	struct {
1273	int client_id;
1274	u32 namelen;
1275	u32 pageslen;
1276	} inbuf;
1277	u32 sc;
1278
1279	args = kcalloc(FASTRPC_CREATE_STATIC_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
1280	if (!args)
1281	return -ENOMEM;
1282
1283	if (copy_from_user(to: &init, from: argp, n: sizeof(init))) {
1284	err = -EFAULT;
1285	goto err;
1286	}
1287
1288	if (init.namelen > INIT_FILE_NAMELEN_MAX) {
1289	err = -EINVAL;
1290	goto err;
1291	}
1292
1293	name = memdup_user(u64_to_user_ptr(init.name), init.namelen);
1294	if (IS_ERR(ptr: name)) {
1295	err = PTR_ERR(ptr: name);
1296	goto err;
1297	}
1298
1299	if (!fl->cctx->remote_heap) {
1300	err = fastrpc_remote_heap_alloc(fl, dev: fl->sctx->dev, size: init.memlen,
1301	obuf: &fl->cctx->remote_heap);
1302	if (err)
1303	goto err_name;
1304
1305	/* Map if we have any heap VMIDs associated with this ADSP Static Process. */
1306	if (fl->cctx->vmcount) {
1307	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
1308
1309	err = qcom_scm_assign_mem(mem_addr: fl->cctx->remote_heap->phys,
1310	mem_sz: (u64)fl->cctx->remote_heap->size,
1311	src: &src_perms,
1312	newvm: fl->cctx->vmperms, dest_cnt: fl->cctx->vmcount);
1313	if (err) {
1314	dev_err(fl->sctx->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n",
1315	fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1316	goto err_map;
1317	}
1318	scm_done = true;
1319	}
1320	}
1321
1322	inbuf.client_id = fl->client_id;
1323	inbuf.namelen = init.namelen;
1324	inbuf.pageslen = 0;
1325	fl->pd = USER_PD;
1326
1327	args[0].ptr = (u64)(uintptr_t)&inbuf;
1328	args[0].length = sizeof(inbuf);
1329	args[0].fd = -1;
1330
1331	args[1].ptr = (u64)(uintptr_t)name;
1332	args[1].length = inbuf.namelen;
1333	args[1].fd = -1;
1334
1335	pages[0].addr = fl->cctx->remote_heap->phys;
1336	pages[0].size = fl->cctx->remote_heap->size;
1337
1338	args[2].ptr = (u64)(uintptr_t) pages;
1339	args[2].length = sizeof(*pages);
1340	args[2].fd = -1;
1341
1342	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_STATIC, 3, 0);
1343
1344	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE,
1345	sc, args);
1346	if (err)
1347	goto err_invoke;
1348
1349	kfree(objp: args);
1350	kfree(objp: name);
1351
1352	return 0;
1353	err_invoke:
1354	if (fl->cctx->vmcount && scm_done) {
1355	u64 src_perms = 0;
1356	struct qcom_scm_vmperm dst_perms;
1357	u32 i;
1358
1359	for (i = 0; i < fl->cctx->vmcount; i++)
1360	src_perms |= BIT(fl->cctx->vmperms[i].vmid);
1361
1362	dst_perms.vmid = QCOM_SCM_VMID_HLOS;
1363	dst_perms.perm = QCOM_SCM_PERM_RWX;
1364	err = qcom_scm_assign_mem(mem_addr: fl->cctx->remote_heap->phys,
1365	mem_sz: (u64)fl->cctx->remote_heap->size,
1366	src: &src_perms, newvm: &dst_perms, dest_cnt: 1);
1367	if (err)
1368	dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n",
1369	fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1370	}
1371	err_map:
1372	fastrpc_buf_free(buf: fl->cctx->remote_heap);
1373	err_name:
1374	kfree(objp: name);
1375	err:
1376	kfree(objp: args);
1377
1378	return err;
1379	}
1380
1381	static int fastrpc_init_create_process(struct fastrpc_user *fl,
1382	char __user *argp)
1383	{
1384	struct fastrpc_init_create init;
1385	struct fastrpc_invoke_args *args;
1386	struct fastrpc_phy_page pages[1];
1387	struct fastrpc_map *map = NULL;
1388	struct fastrpc_buf *imem = NULL;
1389	int memlen;
1390	int err;
1391	struct {
1392	int client_id;
1393	u32 namelen;
1394	u32 filelen;
1395	u32 pageslen;
1396	u32 attrs;
1397	u32 siglen;
1398	} inbuf;
1399	u32 sc;
1400	bool unsigned_module = false;
1401
1402	args = kcalloc(FASTRPC_CREATE_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
1403	if (!args)
1404	return -ENOMEM;
1405
1406	if (copy_from_user(to: &init, from: argp, n: sizeof(init))) {
1407	err = -EFAULT;
1408	goto err;
1409	}
1410
1411	if (init.attrs & FASTRPC_MODE_UNSIGNED_MODULE)
1412	unsigned_module = true;
1413
1414	if (is_session_rejected(fl, unsigned_pd_request: unsigned_module)) {
1415	err = -ECONNREFUSED;
1416	goto err;
1417	}
1418
1419	if (init.filelen > INIT_FILELEN_MAX) {
1420	err = -EINVAL;
1421	goto err;
1422	}
1423
1424	inbuf.client_id = fl->client_id;
1425	inbuf.namelen = strlen(current->comm) + 1;
1426	inbuf.filelen = init.filelen;
1427	inbuf.pageslen = 1;
1428	inbuf.attrs = init.attrs;
1429	inbuf.siglen = init.siglen;
1430	fl->pd = USER_PD;
1431
1432	if (init.filelen && init.filefd) {
1433	err = fastrpc_map_create(fl, fd: init.filefd, len: init.filelen, attr: 0, ppmap: &map);
1434	if (err)
1435	goto err;
1436	}
1437
1438	memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1439	1024 * 1024);
1440	err = fastrpc_buf_alloc(fl, dev: fl->sctx->dev, size: memlen,
1441	obuf: &imem);
1442	if (err)
1443	goto err_alloc;
1444
1445	fl->init_mem = imem;
1446	args[0].ptr = (u64)(uintptr_t)&inbuf;
1447	args[0].length = sizeof(inbuf);
1448	args[0].fd = -1;
1449
1450	args[1].ptr = (u64)(uintptr_t)current->comm;
1451	args[1].length = inbuf.namelen;
1452	args[1].fd = -1;
1453
1454	args[2].ptr = (u64) init.file;
1455	args[2].length = inbuf.filelen;
1456	args[2].fd = init.filefd;
1457
1458	pages[0].addr = imem->phys;
1459	pages[0].size = imem->size;
1460
1461	args[3].ptr = (u64)(uintptr_t) pages;
1462	args[3].length = 1 * sizeof(*pages);
1463	args[3].fd = -1;
1464
1465	args[4].ptr = (u64)(uintptr_t)&inbuf.attrs;
1466	args[4].length = sizeof(inbuf.attrs);
1467	args[4].fd = -1;
1468
1469	args[5].ptr = (u64)(uintptr_t) &inbuf.siglen;
1470	args[5].length = sizeof(inbuf.siglen);
1471	args[5].fd = -1;
1472
1473	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0);
1474	if (init.attrs)
1475	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 4, 0);
1476
1477	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE,
1478	sc, args);
1479	if (err)
1480	goto err_invoke;
1481
1482	kfree(objp: args);
1483
1484	return 0;
1485
1486	err_invoke:
1487	fl->init_mem = NULL;
1488	fastrpc_buf_free(buf: imem);
1489	err_alloc:
1490	fastrpc_map_put(map);
1491	err:
1492	kfree(objp: args);
1493
1494	return err;
1495	}
1496
1497	static struct fastrpc_session_ctx *fastrpc_session_alloc(
1498	struct fastrpc_user *fl)
1499	{
1500	struct fastrpc_channel_ctx *cctx = fl->cctx;
1501	struct fastrpc_session_ctx *session = NULL;
1502	unsigned long flags;
1503	int i;
1504
1505	spin_lock_irqsave(&cctx->lock, flags);
1506	for (i = 0; i < cctx->sesscount; i++) {
1507	if (!cctx->session[i].used && cctx->session[i].valid) {
1508	cctx->session[i].used = true;
1509	session = &cctx->session[i];
1510	/* any non-zero ID will work, session_idx + 1 is the simplest one */
1511	fl->client_id = i + 1;
1512	break;
1513	}
1514	}
1515	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1516
1517	return session;
1518	}
1519
1520	static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,
1521	struct fastrpc_session_ctx *session)
1522	{
1523	unsigned long flags;
1524
1525	spin_lock_irqsave(&cctx->lock, flags);
1526	session->used = false;
1527	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1528	}
1529
1530	static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)
1531	{
1532	struct fastrpc_invoke_args args[1];
1533	int client_id = 0;
1534	u32 sc;
1535
1536	client_id = fl->client_id;
1537	args[0].ptr = (u64)(uintptr_t) &client_id;
1538	args[0].length = sizeof(client_id);
1539	args[0].fd = -1;
1540	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1541
1542	return fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE,
1543	sc, args: &args[0]);
1544	}
1545
1546	static int fastrpc_device_release(struct inode *inode, struct file *file)
1547	{
1548	struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1549	struct fastrpc_channel_ctx *cctx = fl->cctx;
1550	struct fastrpc_invoke_ctx *ctx, *n;
1551	struct fastrpc_map *map, *m;
1552	struct fastrpc_buf *buf, *b;
1553	unsigned long flags;
1554
1555	fastrpc_release_current_dsp_process(fl);
1556
1557	spin_lock_irqsave(&cctx->lock, flags);
1558	list_del(entry: &fl->user);
1559	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1560
1561	if (fl->init_mem)
1562	fastrpc_buf_free(buf: fl->init_mem);
1563
1564	list_for_each_entry_safe(ctx, n, &fl->pending, node) {
1565	list_del(entry: &ctx->node);
1566	fastrpc_context_put(ctx);
1567	}
1568
1569	list_for_each_entry_safe(map, m, &fl->maps, node)
1570	fastrpc_map_put(map);
1571
1572	list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1573	list_del(entry: &buf->node);
1574	fastrpc_buf_free(buf);
1575	}
1576
1577	fastrpc_session_free(cctx, session: fl->sctx);
1578	fastrpc_channel_ctx_put(cctx);
1579
1580	mutex_destroy(lock: &fl->mutex);
1581	kfree(objp: fl);
1582	file->private_data = NULL;
1583
1584	return 0;
1585	}
1586
1587	static int fastrpc_device_open(struct inode *inode, struct file *filp)
1588	{
1589	struct fastrpc_channel_ctx *cctx;
1590	struct fastrpc_device *fdevice;
1591	struct fastrpc_user *fl = NULL;
1592	unsigned long flags;
1593
1594	fdevice = miscdev_to_fdevice(filp->private_data);
1595	cctx = fdevice->cctx;
1596
1597	fl = kzalloc(sizeof(*fl), GFP_KERNEL);
1598	if (!fl)
1599	return -ENOMEM;
1600
1601	/* Released in fastrpc_device_release() */
1602	fastrpc_channel_ctx_get(cctx);
1603
1604	filp->private_data = fl;
1605	spin_lock_init(&fl->lock);
1606	mutex_init(&fl->mutex);
1607	INIT_LIST_HEAD(list: &fl->pending);
1608	INIT_LIST_HEAD(list: &fl->maps);
1609	INIT_LIST_HEAD(list: &fl->mmaps);
1610	INIT_LIST_HEAD(list: &fl->user);
1611	fl->cctx = cctx;
1612	fl->is_secure_dev = fdevice->secure;
1613
1614	fl->sctx = fastrpc_session_alloc(fl);
1615	if (!fl->sctx) {
1616	dev_err(&cctx->rpdev->dev, "No session available\n");
1617	mutex_destroy(lock: &fl->mutex);
1618	kfree(objp: fl);
1619
1620	return -EBUSY;
1621	}
1622
1623	spin_lock_irqsave(&cctx->lock, flags);
1624	list_add_tail(new: &fl->user, head: &cctx->users);
1625	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1626
1627	return 0;
1628	}
1629
1630	static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp)
1631	{
1632	struct fastrpc_alloc_dma_buf bp;
1633	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1634	struct fastrpc_buf *buf = NULL;
1635	int err;
1636
1637	if (copy_from_user(to: &bp, from: argp, n: sizeof(bp)))
1638	return -EFAULT;
1639
1640	err = fastrpc_buf_alloc(fl, dev: fl->sctx->dev, size: bp.size, obuf: &buf);
1641	if (err)
1642	return err;
1643	exp_info.ops = &fastrpc_dma_buf_ops;
1644	exp_info.size = bp.size;
1645	exp_info.flags = O_RDWR;
1646	exp_info.priv = buf;
1647	buf->dmabuf = dma_buf_export(exp_info: &exp_info);
1648	if (IS_ERR(ptr: buf->dmabuf)) {
1649	err = PTR_ERR(ptr: buf->dmabuf);
1650	fastrpc_buf_free(buf);
1651	return err;
1652	}
1653
1654	bp.fd = dma_buf_fd(dmabuf: buf->dmabuf, O_ACCMODE);
1655	if (bp.fd < 0) {
1656	dma_buf_put(dmabuf: buf->dmabuf);
1657	return -EINVAL;
1658	}
1659
1660	if (copy_to_user(to: argp, from: &bp, n: sizeof(bp))) {
1661	/*
1662	* The usercopy failed, but we can't do much about it, as
1663	* dma_buf_fd() already called fd_install() and made the
1664	* file descriptor accessible for the current process. It
1665	* might already be closed and dmabuf no longer valid when
1666	* we reach this point. Therefore "leak" the fd and rely on
1667	* the process exit path to do any required cleanup.
1668	*/
1669	return -EFAULT;
1670	}
1671
1672	return 0;
1673	}
1674
1675	static int fastrpc_init_attach(struct fastrpc_user *fl, int pd)
1676	{
1677	struct fastrpc_invoke_args args[1];
1678	int client_id = fl->client_id;
1679	u32 sc;
1680
1681	args[0].ptr = (u64)(uintptr_t) &client_id;
1682	args[0].length = sizeof(client_id);
1683	args[0].fd = -1;
1684	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1685	fl->pd = pd;
1686
1687	return fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE,
1688	sc, args: &args[0]);
1689	}
1690
1691	static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)
1692	{
1693	struct fastrpc_invoke_args *args = NULL;
1694	struct fastrpc_invoke inv;
1695	u32 nscalars;
1696	int err;
1697
1698	if (copy_from_user(to: &inv, from: argp, n: sizeof(inv)))
1699	return -EFAULT;
1700
1701	/* nscalars is truncated here to max supported value */
1702	nscalars = REMOTE_SCALARS_LENGTH(inv.sc);
1703	if (nscalars) {
1704	args = kcalloc(nscalars, sizeof(*args), GFP_KERNEL);
1705	if (!args)
1706	return -ENOMEM;
1707
1708	if (copy_from_user(to: args, from: (void __user *)(uintptr_t)inv.args,
1709	n: nscalars * sizeof(*args))) {
1710	kfree(objp: args);
1711	return -EFAULT;
1712	}
1713	}
1714
1715	err = fastrpc_internal_invoke(fl, kernel: false, handle: inv.handle, sc: inv.sc, args);
1716	kfree(objp: args);
1717
1718	return err;
1719	}
1720
1721	static int fastrpc_get_info_from_dsp(struct fastrpc_user *fl, uint32_t *dsp_attr_buf,
1722	uint32_t dsp_attr_buf_len)
1723	{
1724	struct fastrpc_invoke_args args[2] = { 0 };
1725
1726	/*
1727	* Capability filled in userspace. This carries the information
1728	* about the remoteproc support which is fetched from the remoteproc
1729	* sysfs node by userspace.
1730	*/
1731	dsp_attr_buf[0] = 0;
1732	dsp_attr_buf_len -= 1;
1733
1734	args[0].ptr = (u64)(uintptr_t)&dsp_attr_buf_len;
1735	args[0].length = sizeof(dsp_attr_buf_len);
1736	args[0].fd = -1;
1737	args[1].ptr = (u64)(uintptr_t)&dsp_attr_buf[1];
1738	args[1].length = dsp_attr_buf_len * sizeof(u32);
1739	args[1].fd = -1;
1740
1741	return fastrpc_internal_invoke(fl, kernel: true, FASTRPC_DSP_UTILITIES_HANDLE,
1742	FASTRPC_SCALARS(0, 1, 1), args);
1743	}
1744
1745	static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap,
1746	struct fastrpc_user *fl)
1747	{
1748	struct fastrpc_channel_ctx *cctx = fl->cctx;
1749	uint32_t attribute_id = cap->attribute_id;
1750	uint32_t *dsp_attributes;
1751	unsigned long flags;
1752	int err;
1753
1754	spin_lock_irqsave(&cctx->lock, flags);
1755	/* check if we already have queried dsp for attributes */
1756	if (cctx->valid_attributes) {
1757	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1758	goto done;
1759	}
1760	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1761
1762	dsp_attributes = kzalloc(FASTRPC_MAX_DSP_ATTRIBUTES_LEN, GFP_KERNEL);
1763	if (!dsp_attributes)
1764	return -ENOMEM;
1765
1766	err = fastrpc_get_info_from_dsp(fl, dsp_attr_buf: dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES);
1767	if (err == DSP_UNSUPPORTED_API) {
1768	dev_info(&cctx->rpdev->dev,
1769	"Warning: DSP capabilities not supported\n");
1770	kfree(objp: dsp_attributes);
1771	return -EOPNOTSUPP;
1772	} else if (err) {
1773	dev_err(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err);
1774	kfree(objp: dsp_attributes);
1775	return err;
1776	}
1777
1778	spin_lock_irqsave(&cctx->lock, flags);
1779	memcpy(cctx->dsp_attributes, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN);
1780	cctx->valid_attributes = true;
1781	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1782	kfree(objp: dsp_attributes);
1783	done:
1784	cap->capability = cctx->dsp_attributes[attribute_id];
1785	return 0;
1786	}
1787
1788	static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp)
1789	{
1790	struct fastrpc_ioctl_capability cap = {0};
1791	int err = 0;
1792
1793	if (copy_from_user(to: &cap, from: argp, n: sizeof(cap)))
1794	return -EFAULT;
1795
1796	cap.capability = 0;
1797
1798	if (cap.attribute_id >= FASTRPC_MAX_DSP_ATTRIBUTES) {
1799	dev_err(&fl->cctx->rpdev->dev, "Error: invalid attribute: %d, err: %d\n",
1800	cap.attribute_id, err);
1801	return -EOVERFLOW;
1802	}
1803
1804	err = fastrpc_get_info_from_kernel(cap: &cap, fl);
1805	if (err)
1806	return err;
1807
1808	if (copy_to_user(to: argp, from: &cap, n: sizeof(cap)))
1809	return -EFAULT;
1810
1811	return 0;
1812	}
1813
1814	static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf *buf)
1815	{
1816	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1817	struct fastrpc_munmap_req_msg req_msg;
1818	struct device *dev = fl->sctx->dev;
1819	int err;
1820	u32 sc;
1821
1822	req_msg.client_id = fl->client_id;
1823	req_msg.size = buf->size;
1824	req_msg.vaddr = buf->raddr;
1825
1826	args[0].ptr = (u64) (uintptr_t) &req_msg;
1827	args[0].length = sizeof(req_msg);
1828
1829	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP, 1, 0);
1830	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE, sc,
1831	args: &args[0]);
1832	if (!err) {
1833	dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr);
1834	spin_lock(lock: &fl->lock);
1835	list_del(entry: &buf->node);
1836	spin_unlock(lock: &fl->lock);
1837	fastrpc_buf_free(buf);
1838	} else {
1839	dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr);
1840	}
1841
1842	return err;
1843	}
1844
1845	static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)
1846	{
1847	struct fastrpc_buf *buf = NULL, *iter, *b;
1848	struct fastrpc_req_munmap req;
1849	struct device *dev = fl->sctx->dev;
1850
1851	if (copy_from_user(to: &req, from: argp, n: sizeof(req)))
1852	return -EFAULT;
1853
1854	spin_lock(lock: &fl->lock);
1855	list_for_each_entry_safe(iter, b, &fl->mmaps, node) {
1856	if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) {
1857	buf = iter;
1858	break;
1859	}
1860	}
1861	spin_unlock(lock: &fl->lock);
1862
1863	if (!buf) {
1864	dev_err(dev, "mmap\t\tpt 0x%09llx [len 0x%08llx] not in list\n",
1865	req.vaddrout, req.size);
1866	return -EINVAL;
1867	}
1868
1869	return fastrpc_req_munmap_impl(fl, buf);
1870	}
1871
1872	static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
1873	{
1874	struct fastrpc_invoke_args args[3] = { [0 ... 2] = { 0 } };
1875	struct fastrpc_buf *buf = NULL;
1876	struct fastrpc_mmap_req_msg req_msg;
1877	struct fastrpc_mmap_rsp_msg rsp_msg;
1878	struct fastrpc_phy_page pages;
1879	struct fastrpc_req_mmap req;
1880	struct device *dev = fl->sctx->dev;
1881	int err;
1882	u32 sc;
1883
1884	if (copy_from_user(to: &req, from: argp, n: sizeof(req)))
1885	return -EFAULT;
1886
1887	if (req.flags != ADSP_MMAP_ADD_PAGES && req.flags != ADSP_MMAP_REMOTE_HEAP_ADDR) {
1888	dev_err(dev, "flag not supported 0x%x\n", req.flags);
1889
1890	return -EINVAL;
1891	}
1892
1893	if (req.vaddrin) {
1894	dev_err(dev, "adding user allocated pages is not supported\n");
1895	return -EINVAL;
1896	}
1897
1898	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR)
1899	err = fastrpc_remote_heap_alloc(fl, dev, size: req.size, obuf: &buf);
1900	else
1901	err = fastrpc_buf_alloc(fl, dev, size: req.size, obuf: &buf);
1902
1903	if (err) {
1904	dev_err(dev, "failed to allocate buffer\n");
1905	return err;
1906	}
1907
1908	req_msg.client_id = fl->client_id;
1909	req_msg.flags = req.flags;
1910	req_msg.vaddr = req.vaddrin;
1911	req_msg.num = sizeof(pages);
1912
1913	args[0].ptr = (u64) (uintptr_t) &req_msg;
1914	args[0].length = sizeof(req_msg);
1915
1916	pages.addr = buf->phys;
1917	pages.size = buf->size;
1918
1919	args[1].ptr = (u64) (uintptr_t) &pages;
1920	args[1].length = sizeof(pages);
1921
1922	args[2].ptr = (u64) (uintptr_t) &rsp_msg;
1923	args[2].length = sizeof(rsp_msg);
1924
1925	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP, 2, 1);
1926	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE, sc,
1927	args: &args[0]);
1928	if (err) {
1929	dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size);
1930	fastrpc_buf_free(buf);
1931	return err;
1932	}
1933
1934	/* update the buffer to be able to deallocate the memory on the DSP */
1935	buf->raddr = (uintptr_t) rsp_msg.vaddr;
1936
1937	/* let the client know the address to use */
1938	req.vaddrout = rsp_msg.vaddr;
1939
1940	/* Add memory to static PD pool, protection thru hypervisor */
1941	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR && fl->cctx->vmcount) {
1942	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
1943
1944	err = qcom_scm_assign_mem(mem_addr: buf->phys, mem_sz: (u64)buf->size,
1945	src: &src_perms, newvm: fl->cctx->vmperms, dest_cnt: fl->cctx->vmcount);
1946	if (err) {
1947	dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
1948	buf->phys, buf->size, err);
1949	goto err_assign;
1950	}
1951	}
1952
1953	spin_lock(lock: &fl->lock);
1954	list_add_tail(new: &buf->node, head: &fl->mmaps);
1955	spin_unlock(lock: &fl->lock);
1956
1957	if (copy_to_user(to: (void __user *)argp, from: &req, n: sizeof(req))) {
1958	err = -EFAULT;
1959	goto err_assign;
1960	}
1961
1962	dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n",
1963	buf->raddr, buf->size);
1964
1965	return 0;
1966
1967	err_assign:
1968	fastrpc_req_munmap_impl(fl, buf);
1969
1970	return err;
1971	}
1972
1973	static int fastrpc_req_mem_unmap_impl(struct fastrpc_user *fl, struct fastrpc_mem_unmap *req)
1974	{
1975	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1976	struct fastrpc_map *map = NULL, *iter, *m;
1977	struct fastrpc_mem_unmap_req_msg req_msg = { 0 };
1978	int err = 0;
1979	u32 sc;
1980	struct device *dev = fl->sctx->dev;
1981
1982	spin_lock(lock: &fl->lock);
1983	list_for_each_entry_safe(iter, m, &fl->maps, node) {
1984	if ((req->fd < 0 || iter->fd == req->fd) && (iter->raddr == req->vaddr)) {
1985	map = iter;
1986	break;
1987	}
1988	}
1989
1990	spin_unlock(lock: &fl->lock);
1991
1992	if (!map) {
1993	dev_err(dev, "map not in list\n");
1994	return -EINVAL;
1995	}
1996
1997	req_msg.client_id = fl->client_id;
1998	req_msg.len = map->len;
1999	req_msg.vaddrin = map->raddr;
2000	req_msg.fd = map->fd;
2001
2002	args[0].ptr = (u64) (uintptr_t) &req_msg;
2003	args[0].length = sizeof(req_msg);
2004
2005	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_UNMAP, 1, 0);
2006	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE, sc,
2007	args: &args[0]);
2008	if (err) {
2009	dev_err(dev, "unmmap\tpt fd = %d, 0x%09llx error\n", map->fd, map->raddr);
2010	return err;
2011	}
2012	fastrpc_map_put(map);
2013
2014	return 0;
2015	}
2016
2017	static int fastrpc_req_mem_unmap(struct fastrpc_user *fl, char __user *argp)
2018	{
2019	struct fastrpc_mem_unmap req;
2020
2021	if (copy_from_user(to: &req, from: argp, n: sizeof(req)))
2022	return -EFAULT;
2023
2024	return fastrpc_req_mem_unmap_impl(fl, req: &req);
2025	}
2026
2027	static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp)
2028	{
2029	struct fastrpc_invoke_args args[4] = { [0 ... 3] = { 0 } };
2030	struct fastrpc_mem_map_req_msg req_msg = { 0 };
2031	struct fastrpc_mmap_rsp_msg rsp_msg = { 0 };
2032	struct fastrpc_mem_unmap req_unmap = { 0 };
2033	struct fastrpc_phy_page pages = { 0 };
2034	struct fastrpc_mem_map req;
2035	struct device *dev = fl->sctx->dev;
2036	struct fastrpc_map *map = NULL;
2037	int err;
2038	u32 sc;
2039
2040	if (copy_from_user(to: &req, from: argp, n: sizeof(req)))
2041	return -EFAULT;
2042
2043	/* create SMMU mapping */
2044	err = fastrpc_map_create(fl, fd: req.fd, len: req.length, attr: 0, ppmap: &map);
2045	if (err) {
2046	dev_err(dev, "failed to map buffer, fd = %d\n", req.fd);
2047	return err;
2048	}
2049
2050	req_msg.client_id = fl->client_id;
2051	req_msg.fd = req.fd;
2052	req_msg.offset = req.offset;
2053	req_msg.vaddrin = req.vaddrin;
2054	map->va = (void *) (uintptr_t) req.vaddrin;
2055	req_msg.flags = req.flags;
2056	req_msg.num = sizeof(pages);
2057	req_msg.data_len = 0;
2058
2059	args[0].ptr = (u64) (uintptr_t) &req_msg;
2060	args[0].length = sizeof(req_msg);
2061
2062	pages.addr = map->phys;
2063	pages.size = map->len;
2064
2065	args[1].ptr = (u64) (uintptr_t) &pages;
2066	args[1].length = sizeof(pages);
2067
2068	args[2].ptr = (u64) (uintptr_t) &pages;
2069	args[2].length = 0;
2070
2071	args[3].ptr = (u64) (uintptr_t) &rsp_msg;
2072	args[3].length = sizeof(rsp_msg);
2073
2074	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_MAP, 3, 1);
2075	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE, sc, args: &args[0]);
2076	if (err) {
2077	dev_err(dev, "mem mmap error, fd %d, vaddr %llx, size %lld\n",
2078	req.fd, req.vaddrin, map->len);
2079	goto err_invoke;
2080	}
2081
2082	/* update the buffer to be able to deallocate the memory on the DSP */
2083	map->raddr = rsp_msg.vaddr;
2084
2085	/* let the client know the address to use */
2086	req.vaddrout = rsp_msg.vaddr;
2087
2088	if (copy_to_user(to: (void __user *)argp, from: &req, n: sizeof(req))) {
2089	/* unmap the memory and release the buffer */
2090	req_unmap.vaddr = (uintptr_t) rsp_msg.vaddr;
2091	req_unmap.length = map->len;
2092	fastrpc_req_mem_unmap_impl(fl, req: &req_unmap);
2093	return -EFAULT;
2094	}
2095
2096	return 0;
2097
2098	err_invoke:
2099	fastrpc_map_put(map);
2100
2101	return err;
2102	}
2103
2104	static long fastrpc_device_ioctl(struct file *file, unsigned int cmd,
2105	unsigned long arg)
2106	{
2107	struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
2108	char __user *argp = (char __user *)arg;
2109	int err;
2110
2111	switch (cmd) {
2112	case FASTRPC_IOCTL_INVOKE:
2113	err = fastrpc_invoke(fl, argp);
2114	break;
2115	case FASTRPC_IOCTL_INIT_ATTACH:
2116	err = fastrpc_init_attach(fl, ROOT_PD);
2117	break;
2118	case FASTRPC_IOCTL_INIT_ATTACH_SNS:
2119	err = fastrpc_init_attach(fl, SENSORS_PD);
2120	break;
2121	case FASTRPC_IOCTL_INIT_CREATE_STATIC:
2122	err = fastrpc_init_create_static_process(fl, argp);
2123	break;
2124	case FASTRPC_IOCTL_INIT_CREATE:
2125	err = fastrpc_init_create_process(fl, argp);
2126	break;
2127	case FASTRPC_IOCTL_ALLOC_DMA_BUFF:
2128	err = fastrpc_dmabuf_alloc(fl, argp);
2129	break;
2130	case FASTRPC_IOCTL_MMAP:
2131	err = fastrpc_req_mmap(fl, argp);
2132	break;
2133	case FASTRPC_IOCTL_MUNMAP:
2134	err = fastrpc_req_munmap(fl, argp);
2135	break;
2136	case FASTRPC_IOCTL_MEM_MAP:
2137	err = fastrpc_req_mem_map(fl, argp);
2138	break;
2139	case FASTRPC_IOCTL_MEM_UNMAP:
2140	err = fastrpc_req_mem_unmap(fl, argp);
2141	break;
2142	case FASTRPC_IOCTL_GET_DSP_INFO:
2143	err = fastrpc_get_dsp_info(fl, argp);
2144	break;
2145	default:
2146	err = -ENOTTY;
2147	break;
2148	}
2149
2150	return err;
2151	}
2152
2153	static const struct file_operations fastrpc_fops = {
2154	.open = fastrpc_device_open,
2155	.release = fastrpc_device_release,
2156	.unlocked_ioctl = fastrpc_device_ioctl,
2157	.compat_ioctl = fastrpc_device_ioctl,
2158	};
2159
2160	static int fastrpc_cb_probe(struct platform_device *pdev)
2161	{
2162	struct fastrpc_channel_ctx *cctx;
2163	struct fastrpc_session_ctx *sess;
2164	struct device *dev = &pdev->dev;
2165	int i, sessions = 0;
2166	unsigned long flags;
2167	int rc;
2168
2169	cctx = dev_get_drvdata(dev: dev->parent);
2170	if (!cctx)
2171	return -EINVAL;
2172
2173	of_property_read_u32(np: dev->of_node, propname: "qcom,nsessions", out_value: &sessions);
2174
2175	spin_lock_irqsave(&cctx->lock, flags);
2176	if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) {
2177	dev_err(&pdev->dev, "too many sessions\n");
2178	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2179	return -ENOSPC;
2180	}
2181	sess = &cctx->session[cctx->sesscount++];
2182	sess->used = false;
2183	sess->valid = true;
2184	sess->dev = dev;
2185	dev_set_drvdata(dev, data: sess);
2186
2187	if (of_property_read_u32(np: dev->of_node, propname: "reg", out_value: &sess->sid))
2188	dev_info(dev, "FastRPC Session ID not specified in DT\n");
2189
2190	if (sessions > 0) {
2191	struct fastrpc_session_ctx *dup_sess;
2192
2193	for (i = 1; i < sessions; i++) {
2194	if (cctx->sesscount >= FASTRPC_MAX_SESSIONS)
2195	break;
2196	dup_sess = &cctx->session[cctx->sesscount++];
2197	memcpy(dup_sess, sess, sizeof(*dup_sess));
2198	}
2199	}
2200	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2201	rc = dma_set_mask(dev, DMA_BIT_MASK(32));
2202	if (rc) {
2203	dev_err(dev, "32-bit DMA enable failed\n");
2204	return rc;
2205	}
2206
2207	return 0;
2208	}
2209
2210	static void fastrpc_cb_remove(struct platform_device *pdev)
2211	{
2212	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(dev: pdev->dev.parent);
2213	struct fastrpc_session_ctx *sess = dev_get_drvdata(dev: &pdev->dev);
2214	unsigned long flags;
2215	int i;
2216
2217	spin_lock_irqsave(&cctx->lock, flags);
2218	for (i = 0; i < FASTRPC_MAX_SESSIONS; i++) {
2219	if (cctx->session[i].sid == sess->sid) {
2220	cctx->session[i].valid = false;
2221	cctx->sesscount--;
2222	}
2223	}
2224	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2225	}
2226
2227	static const struct of_device_id fastrpc_match_table[] = {
2228	{ .compatible = "qcom,fastrpc-compute-cb", },
2229	{}
2230	};
2231
2232	static struct platform_driver fastrpc_cb_driver = {
2233	.probe = fastrpc_cb_probe,
2234	.remove = fastrpc_cb_remove,
2235	.driver = {
2236	.name = "qcom,fastrpc-cb",
2237	.of_match_table = fastrpc_match_table,
2238	.suppress_bind_attrs = true,
2239	},
2240	};
2241
2242	static int fastrpc_device_register(struct device *dev, struct fastrpc_channel_ctx *cctx,
2243	bool is_secured, const char *domain)
2244	{
2245	struct fastrpc_device *fdev;
2246	int err;
2247
2248	fdev = devm_kzalloc(dev, size: sizeof(*fdev), GFP_KERNEL);
2249	if (!fdev)
2250	return -ENOMEM;
2251
2252	fdev->secure = is_secured;
2253	fdev->cctx = cctx;
2254	fdev->miscdev.minor = MISC_DYNAMIC_MINOR;
2255	fdev->miscdev.fops = &fastrpc_fops;
2256	fdev->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, fmt: "fastrpc-%s%s",
2257	domain, is_secured ? "-secure" : "");
2258	if (!fdev->miscdev.name)
2259	return -ENOMEM;
2260
2261	err = misc_register(misc: &fdev->miscdev);
2262	if (!err) {
2263	if (is_secured)
2264	cctx->secure_fdevice = fdev;
2265	else
2266	cctx->fdevice = fdev;
2267	}
2268
2269	return err;
2270	}
2271
2272	static int fastrpc_get_domain_id(const char *domain)
2273	{
2274	if (!strncmp(domain, "adsp", 4))
2275	return ADSP_DOMAIN_ID;
2276	else if (!strncmp(domain, "cdsp", 4))
2277	return CDSP_DOMAIN_ID;
2278	else if (!strncmp(domain, "mdsp", 4))
2279	return MDSP_DOMAIN_ID;
2280	else if (!strncmp(domain, "sdsp", 4))
2281	return SDSP_DOMAIN_ID;
2282	else if (!strncmp(domain, "gdsp", 4))
2283	return GDSP_DOMAIN_ID;
2284
2285	return -EINVAL;
2286	}
2287
2288	static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
2289	{
2290	struct device *rdev = &rpdev->dev;
2291	struct fastrpc_channel_ctx *data;
2292	int i, err, domain_id = -1, vmcount;
2293	const char *domain;
2294	bool secure_dsp;
2295	unsigned int vmids[FASTRPC_MAX_VMIDS];
2296
2297	err = of_property_read_string(np: rdev->of_node, propname: "label", out_string: &domain);
2298	if (err) {
2299	dev_info(rdev, "FastRPC Domain not specified in DT\n");
2300	return err;
2301	}
2302
2303	domain_id = fastrpc_get_domain_id(domain);
2304
2305	if (domain_id < 0) {
2306	dev_info(rdev, "FastRPC Domain %s not supported\n", domain);
2307	return -EINVAL;
2308	}
2309
2310	if (of_reserved_mem_device_init_by_idx(dev: rdev, np: rdev->of_node, idx: 0))
2311	dev_info(rdev, "no reserved DMA memory for FASTRPC\n");
2312
2313	vmcount = of_property_read_variable_u32_array(np: rdev->of_node,
2314	propname: "qcom,vmids", out_values: &vmids[0], sz_min: 0, FASTRPC_MAX_VMIDS);
2315	if (vmcount < 0)
2316	vmcount = 0;
2317	else if (!qcom_scm_is_available())
2318	return -EPROBE_DEFER;
2319
2320	data = kzalloc(sizeof(*data), GFP_KERNEL);
2321	if (!data)
2322	return -ENOMEM;
2323
2324	if (vmcount) {
2325	data->vmcount = vmcount;
2326	for (i = 0; i < data->vmcount; i++) {
2327	data->vmperms[i].vmid = vmids[i];
2328	data->vmperms[i].perm = QCOM_SCM_PERM_RWX;
2329	}
2330	}
2331
2332	if (domain_id == SDSP_DOMAIN_ID) {
2333	struct resource res;
2334	u64 src_perms;
2335
2336	err = of_reserved_mem_region_to_resource(np: rdev->of_node, idx: 0, res: &res);
2337	if (!err) {
2338	src_perms = BIT(QCOM_SCM_VMID_HLOS);
2339
2340	qcom_scm_assign_mem(mem_addr: res.start, mem_sz: resource_size(res: &res), src: &src_perms,
2341	newvm: data->vmperms, dest_cnt: data->vmcount);
2342	}
2343
2344	}
2345
2346	secure_dsp = !(of_property_read_bool(np: rdev->of_node, propname: "qcom,non-secure-domain"));
2347	data->secure = secure_dsp;
2348
2349	switch (domain_id) {
2350	case ADSP_DOMAIN_ID:
2351	case MDSP_DOMAIN_ID:
2352	case SDSP_DOMAIN_ID:
2353	/* Unsigned PD offloading is only supported on CDSP and GDSP */
2354	data->unsigned_support = false;
2355	err = fastrpc_device_register(dev: rdev, cctx: data, is_secured: secure_dsp, domain);
2356	if (err)
2357	goto err_free_data;
2358	break;
2359	case CDSP_DOMAIN_ID:
2360	case GDSP_DOMAIN_ID:
2361	data->unsigned_support = true;
2362	/* Create both device nodes so that we can allow both Signed and Unsigned PD */
2363	err = fastrpc_device_register(dev: rdev, cctx: data, is_secured: true, domain);
2364	if (err)
2365	goto err_free_data;
2366
2367	err = fastrpc_device_register(dev: rdev, cctx: data, is_secured: false, domain);
2368	if (err)
2369	goto err_deregister_fdev;
2370	break;
2371	default:
2372	err = -EINVAL;
2373	goto err_free_data;
2374	}
2375
2376	kref_init(kref: &data->refcount);
2377
2378	dev_set_drvdata(dev: &rpdev->dev, data);
2379	rdev->dma_mask = &data->dma_mask;
2380	dma_set_mask_and_coherent(dev: rdev, DMA_BIT_MASK(32));
2381	INIT_LIST_HEAD(list: &data->users);
2382	INIT_LIST_HEAD(list: &data->invoke_interrupted_mmaps);
2383	spin_lock_init(&data->lock);
2384	idr_init(idr: &data->ctx_idr);
2385	data->domain_id = domain_id;
2386	data->rpdev = rpdev;
2387
2388	err = of_platform_populate(root: rdev->of_node, NULL, NULL, parent: rdev);
2389	if (err)
2390	goto err_deregister_fdev;
2391
2392	return 0;
2393
2394	err_deregister_fdev:
2395	if (data->fdevice)
2396	misc_deregister(misc: &data->fdevice->miscdev);
2397	if (data->secure_fdevice)
2398	misc_deregister(misc: &data->secure_fdevice->miscdev);
2399
2400	err_free_data:
2401	kfree(objp: data);
2402	return err;
2403	}
2404
2405	static void fastrpc_notify_users(struct fastrpc_user *user)
2406	{
2407	struct fastrpc_invoke_ctx *ctx;
2408
2409	spin_lock(lock: &user->lock);
2410	list_for_each_entry(ctx, &user->pending, node) {
2411	ctx->retval = -EPIPE;
2412	complete(&ctx->work);
2413	}
2414	spin_unlock(lock: &user->lock);
2415	}
2416
2417	static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
2418	{
2419	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(dev: &rpdev->dev);
2420	struct fastrpc_buf *buf, *b;
2421	struct fastrpc_user *user;
2422	unsigned long flags;
2423
2424	/* No invocations past this point */
2425	spin_lock_irqsave(&cctx->lock, flags);
2426	cctx->rpdev = NULL;
2427	list_for_each_entry(user, &cctx->users, user)
2428	fastrpc_notify_users(user);
2429	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2430
2431	if (cctx->fdevice)
2432	misc_deregister(misc: &cctx->fdevice->miscdev);
2433
2434	if (cctx->secure_fdevice)
2435	misc_deregister(misc: &cctx->secure_fdevice->miscdev);
2436
2437	list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node)
2438	list_del(entry: &buf->node);
2439
2440	if (cctx->remote_heap)
2441	fastrpc_buf_free(buf: cctx->remote_heap);
2442
2443	of_platform_depopulate(parent: &rpdev->dev);
2444
2445	fastrpc_channel_ctx_put(cctx);
2446	}
2447
2448	static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,
2449	int len, void *priv, u32 addr)
2450	{
2451	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(dev: &rpdev->dev);
2452	struct fastrpc_invoke_rsp *rsp = data;
2453	struct fastrpc_invoke_ctx *ctx;
2454	unsigned long flags;
2455	unsigned long ctxid;
2456
2457	if (len < sizeof(*rsp))
2458	return -EINVAL;
2459
2460	ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4);
2461
2462	spin_lock_irqsave(&cctx->lock, flags);
2463	ctx = idr_find(&cctx->ctx_idr, id: ctxid);
2464	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2465
2466	if (!ctx) {
2467	dev_err(&rpdev->dev, "No context ID matches response\n");
2468	return -ENOENT;
2469	}
2470
2471	ctx->retval = rsp->retval;
2472	complete(&ctx->work);
2473
2474	/*
2475	* The DMA buffer associated with the context cannot be freed in
2476	* interrupt context so schedule it through a worker thread to
2477	* avoid a kernel BUG.
2478	*/
2479	schedule_work(work: &ctx->put_work);
2480
2481	return 0;
2482	}
2483
2484	static const struct of_device_id fastrpc_rpmsg_of_match[] = {
2485	{ .compatible = "qcom,fastrpc" },
2486	{ },
2487	};
2488	MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);
2489
2490	static struct rpmsg_driver fastrpc_driver = {
2491	.probe = fastrpc_rpmsg_probe,
2492	.remove = fastrpc_rpmsg_remove,
2493	.callback = fastrpc_rpmsg_callback,
2494	.drv = {
2495	.name = "qcom,fastrpc",
2496	.of_match_table = fastrpc_rpmsg_of_match,
2497	},
2498	};
2499
2500	static int fastrpc_init(void)
2501	{
2502	int ret;
2503
2504	ret = platform_driver_register(&fastrpc_cb_driver);
2505	if (ret < 0) {
2506	pr_err("fastrpc: failed to register cb driver\n");
2507	return ret;
2508	}
2509
2510	ret = register_rpmsg_driver(&fastrpc_driver);
2511	if (ret < 0) {
2512	pr_err("fastrpc: failed to register rpmsg driver\n");
2513	platform_driver_unregister(&fastrpc_cb_driver);
2514	return ret;
2515	}
2516
2517	return 0;
2518	}
2519	module_init(fastrpc_init);
2520
2521	static void fastrpc_exit(void)
2522	{
2523	platform_driver_unregister(&fastrpc_cb_driver);
2524	unregister_rpmsg_driver(drv: &fastrpc_driver);
2525	}
2526	module_exit(fastrpc_exit);
2527
2528	MODULE_DESCRIPTION("Qualcomm FastRPC");
2529	MODULE_LICENSE("GPL v2");
2530	MODULE_IMPORT_NS("DMA_BUF");
2531