1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2016-2017, Linaro Ltd
4 */
5
6#include <linux/idr.h>
7#include <linux/interrupt.h>
8#include <linux/io.h>
9#include <linux/list.h>
10#include <linux/mfd/syscon.h>
11#include <linux/module.h>
12#include <linux/of.h>
13#include <linux/of_address.h>
14#include <linux/platform_device.h>
15#include <linux/regmap.h>
16#include <linux/rpmsg.h>
17#include <linux/sizes.h>
18#include <linux/slab.h>
19#include <linux/wait.h>
20#include <linux/workqueue.h>
21#include <linux/mailbox_client.h>
22
23#include "rpmsg_internal.h"
24#include "qcom_glink_native.h"
25
26#define CREATE_TRACE_POINTS
27#include "qcom_glink_trace.h"
28
29#define GLINK_NAME_SIZE 32
30#define GLINK_VERSION_1 1
31
32#define RPM_GLINK_CID_MIN 1
33#define RPM_GLINK_CID_MAX 65536
34
35struct glink_msg {
36 /* New members MUST be added within the __struct_group() macro below. */
37 __struct_group(glink_msg_hdr, hdr, __packed,
38 __le16 cmd;
39 __le16 param1;
40 __le32 param2;
41 );
42 u8 data[];
43} __packed;
44static_assert(offsetof(struct glink_msg, data) == sizeof(struct glink_msg_hdr),
45 "struct member likely outside of __struct_group()");
46
47/**
48 * struct glink_defer_cmd - deferred incoming control message
49 * @node: list node
50 * @msg: message header
51 * @data: payload of the message
52 *
53 * Copy of a received control message, to be added to @rx_queue and processed
54 * by @rx_work of @qcom_glink.
55 */
56struct glink_defer_cmd {
57 struct list_head node;
58
59 struct glink_msg_hdr msg;
60 u8 data[];
61};
62
63/**
64 * struct glink_core_rx_intent - RX intent
65 * RX intent
66 *
67 * @data: pointer to the data (may be NULL for zero-copy)
68 * @id: remote or local intent ID
69 * @size: size of the original intent (do not modify)
70 * @reuse: To mark if the intent can be reused after first use
71 * @in_use: To mark if intent is already in use for the channel
72 * @offset: next write offset (initially 0)
73 * @node: list node
74 */
75struct glink_core_rx_intent {
76 void *data;
77 u32 id;
78 size_t size;
79 bool reuse;
80 bool in_use;
81 u32 offset;
82
83 struct list_head node;
84};
85
86/**
87 * struct qcom_glink - driver context, relates to one remote subsystem
88 * @dev: reference to the associated struct device
89 * @label: identifier of the glink edge
90 * @rx_pipe: pipe object for receive FIFO
91 * @tx_pipe: pipe object for transmit FIFO
92 * @rx_work: worker for handling received control messages
93 * @rx_lock: protects the @rx_queue
94 * @rx_queue: queue of received control messages to be processed in @rx_work
95 * @tx_lock: synchronizes operations on the tx fifo
96 * @idr_lock: synchronizes @lcids and @rcids modifications
97 * @lcids: idr of all channels with a known local channel id
98 * @rcids: idr of all channels with a known remote channel id
99 * @features: remote features
100 * @intentless: flag to indicate that there is no intent
101 * @tx_avail_notify: Waitqueue for pending tx tasks
102 * @sent_read_notify: flag to check cmd sent or not
103 * @abort_tx: flag indicating that all tx attempts should fail
104 */
105struct qcom_glink {
106 struct device *dev;
107
108 const char *label;
109
110 struct qcom_glink_pipe *rx_pipe;
111 struct qcom_glink_pipe *tx_pipe;
112
113 struct work_struct rx_work;
114 spinlock_t rx_lock;
115 struct list_head rx_queue;
116
117 spinlock_t tx_lock;
118
119 spinlock_t idr_lock;
120 struct idr lcids;
121 struct idr rcids;
122 unsigned long features;
123
124 bool intentless;
125 wait_queue_head_t tx_avail_notify;
126 bool sent_read_notify;
127
128 bool abort_tx;
129};
130
131enum {
132 GLINK_STATE_CLOSED,
133 GLINK_STATE_OPENING,
134 GLINK_STATE_OPEN,
135 GLINK_STATE_CLOSING,
136};
137
138/**
139 * struct glink_channel - internal representation of a channel
140 * @rpdev: rpdev reference, only used for primary endpoints
141 * @ept: rpmsg endpoint this channel is associated with
142 * @glink: qcom_glink context handle
143 * @refcount: refcount for the channel object
144 * @recv_lock: guard for @ept.cb
145 * @name: unique channel name/identifier
146 * @lcid: channel id, in local space
147 * @rcid: channel id, in remote space
148 * @intent_lock: lock for protection of @liids, @riids
149 * @liids: idr of all local intents
150 * @riids: idr of all remote intents
151 * @intent_work: worker responsible for transmitting rx_done packets
152 * @done_intents: list of intents that needs to be announced rx_done
153 * @buf: receive buffer, for gathering fragments
154 * @buf_offset: write offset in @buf
155 * @buf_size: size of current @buf
156 * @open_ack: completed once remote has acked the open-request
157 * @open_req: completed once open-request has been received
158 * @intent_req_lock: Synchronises multiple intent requests
159 * @intent_req_result: Result of intent request
160 * @intent_received: flag indicating that an intent has been received
161 * @intent_req_wq: wait queue for intent_req signalling
162 */
163struct glink_channel {
164 struct rpmsg_endpoint ept;
165
166 struct rpmsg_device *rpdev;
167 struct qcom_glink *glink;
168
169 struct kref refcount;
170
171 spinlock_t recv_lock;
172
173 char *name;
174 unsigned int lcid;
175 unsigned int rcid;
176
177 spinlock_t intent_lock;
178 struct idr liids;
179 struct idr riids;
180 struct work_struct intent_work;
181 struct list_head done_intents;
182
183 struct glink_core_rx_intent *buf;
184 int buf_offset;
185 int buf_size;
186
187 struct completion open_ack;
188 struct completion open_req;
189
190 struct mutex intent_req_lock;
191 int intent_req_result;
192 bool intent_received;
193 wait_queue_head_t intent_req_wq;
194};
195
196#define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept)
197
198static const struct rpmsg_endpoint_ops glink_endpoint_ops;
199
200#define GLINK_CMD_VERSION 0
201#define GLINK_CMD_VERSION_ACK 1
202#define GLINK_CMD_OPEN 2
203#define GLINK_CMD_CLOSE 3
204#define GLINK_CMD_OPEN_ACK 4
205#define GLINK_CMD_INTENT 5
206#define GLINK_CMD_RX_DONE 6
207#define GLINK_CMD_RX_INTENT_REQ 7
208#define GLINK_CMD_RX_INTENT_REQ_ACK 8
209#define GLINK_CMD_TX_DATA 9
210#define GLINK_CMD_CLOSE_ACK 11
211#define GLINK_CMD_TX_DATA_CONT 12
212#define GLINK_CMD_READ_NOTIF 13
213#define GLINK_CMD_RX_DONE_W_REUSE 14
214#define GLINK_CMD_SIGNALS 15
215
216#define GLINK_FEATURE_INTENTLESS BIT(1)
217
218#define NATIVE_DTR_SIG NATIVE_DSR_SIG
219#define NATIVE_DSR_SIG BIT(31)
220#define NATIVE_RTS_SIG NATIVE_CTS_SIG
221#define NATIVE_CTS_SIG BIT(30)
222
223static void qcom_glink_rx_done_work(struct work_struct *work);
224
225static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink,
226 const char *name)
227{
228 struct glink_channel *channel;
229
230 channel = kzalloc(sizeof(*channel), GFP_KERNEL);
231 if (!channel)
232 return ERR_PTR(error: -ENOMEM);
233
234 /* Setup glink internal glink_channel data */
235 spin_lock_init(&channel->recv_lock);
236 spin_lock_init(&channel->intent_lock);
237 mutex_init(&channel->intent_req_lock);
238
239 channel->glink = glink;
240 channel->name = kstrdup(s: name, GFP_KERNEL);
241 if (!channel->name) {
242 kfree(objp: channel);
243 return ERR_PTR(error: -ENOMEM);
244 }
245
246 init_completion(x: &channel->open_req);
247 init_completion(x: &channel->open_ack);
248 init_waitqueue_head(&channel->intent_req_wq);
249
250 INIT_LIST_HEAD(list: &channel->done_intents);
251 INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work);
252
253 idr_init(idr: &channel->liids);
254 idr_init(idr: &channel->riids);
255 kref_init(kref: &channel->refcount);
256
257 return channel;
258}
259
260static void qcom_glink_channel_release(struct kref *ref)
261{
262 struct glink_channel *channel = container_of(ref, struct glink_channel,
263 refcount);
264 struct glink_core_rx_intent *intent;
265 struct glink_core_rx_intent *tmp;
266 unsigned long flags;
267 int iid;
268
269 /* cancel pending rx_done work */
270 cancel_work_sync(work: &channel->intent_work);
271
272 spin_lock_irqsave(&channel->intent_lock, flags);
273 /* Free all non-reuse intents pending rx_done work */
274 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
275 if (!intent->reuse) {
276 kfree(objp: intent->data);
277 kfree(objp: intent);
278 }
279 }
280
281 idr_for_each_entry(&channel->liids, tmp, iid) {
282 kfree(objp: tmp->data);
283 kfree(objp: tmp);
284 }
285 idr_destroy(&channel->liids);
286
287 idr_for_each_entry(&channel->riids, tmp, iid)
288 kfree(objp: tmp);
289 idr_destroy(&channel->riids);
290 spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
291
292 kfree(objp: channel->name);
293 kfree(objp: channel);
294}
295
296static size_t qcom_glink_rx_avail(struct qcom_glink *glink)
297{
298 return glink->rx_pipe->avail(glink->rx_pipe);
299}
300
301static void qcom_glink_rx_peek(struct qcom_glink *glink,
302 void *data, unsigned int offset, size_t count)
303{
304 glink->rx_pipe->peek(glink->rx_pipe, data, offset, count);
305}
306
307static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count)
308{
309 glink->rx_pipe->advance(glink->rx_pipe, count);
310}
311
312static size_t qcom_glink_tx_avail(struct qcom_glink *glink)
313{
314 return glink->tx_pipe->avail(glink->tx_pipe);
315}
316
317static void qcom_glink_tx_write(struct qcom_glink *glink,
318 const void *hdr, size_t hlen,
319 const void *data, size_t dlen)
320{
321 glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen);
322}
323
324static void qcom_glink_tx_kick(struct qcom_glink *glink)
325{
326 glink->tx_pipe->kick(glink->tx_pipe);
327}
328
329static void qcom_glink_send_read_notify(struct qcom_glink *glink)
330{
331 struct glink_msg msg;
332
333 msg.cmd = cpu_to_le16(GLINK_CMD_READ_NOTIF);
334 msg.param1 = 0;
335 msg.param2 = 0;
336
337 qcom_glink_tx_write(glink, hdr: &msg, hlen: sizeof(msg), NULL, dlen: 0);
338
339 qcom_glink_tx_kick(glink);
340}
341
342static int qcom_glink_tx(struct qcom_glink *glink,
343 const void *hdr, size_t hlen,
344 const void *data, size_t dlen, bool wait)
345{
346 unsigned int tlen = hlen + dlen;
347 unsigned long flags;
348 int ret = 0;
349
350 /* Reject packets that are too big */
351 if (tlen >= glink->tx_pipe->length)
352 return -EINVAL;
353
354 spin_lock_irqsave(&glink->tx_lock, flags);
355
356 if (glink->abort_tx) {
357 ret = -EIO;
358 goto out;
359 }
360
361 while (qcom_glink_tx_avail(glink) < tlen) {
362 if (!wait) {
363 ret = -EAGAIN;
364 goto out;
365 }
366
367 if (glink->abort_tx) {
368 ret = -EIO;
369 goto out;
370 }
371
372 if (!glink->sent_read_notify) {
373 glink->sent_read_notify = true;
374 qcom_glink_send_read_notify(glink);
375 }
376
377 /* Wait without holding the tx_lock */
378 spin_unlock_irqrestore(lock: &glink->tx_lock, flags);
379
380 wait_event_timeout(glink->tx_avail_notify,
381 qcom_glink_tx_avail(glink) >= tlen, 10 * HZ);
382
383 spin_lock_irqsave(&glink->tx_lock, flags);
384
385 if (qcom_glink_tx_avail(glink) >= tlen)
386 glink->sent_read_notify = false;
387 }
388
389 qcom_glink_tx_write(glink, hdr, hlen, data, dlen);
390 qcom_glink_tx_kick(glink);
391
392out:
393 spin_unlock_irqrestore(lock: &glink->tx_lock, flags);
394
395 return ret;
396}
397
398static int qcom_glink_send_version(struct qcom_glink *glink)
399{
400 struct glink_msg msg;
401
402 msg.cmd = cpu_to_le16(GLINK_CMD_VERSION);
403 msg.param1 = cpu_to_le16(GLINK_VERSION_1);
404 msg.param2 = cpu_to_le32(glink->features);
405
406 trace_qcom_glink_cmd_version_tx(glink->label, GLINK_VERSION_1, glink->features);
407
408 return qcom_glink_tx(glink, hdr: &msg, hlen: sizeof(msg), NULL, dlen: 0, wait: true);
409}
410
411static void qcom_glink_send_version_ack(struct qcom_glink *glink)
412{
413 struct glink_msg msg;
414
415 msg.cmd = cpu_to_le16(GLINK_CMD_VERSION_ACK);
416 msg.param1 = cpu_to_le16(GLINK_VERSION_1);
417 msg.param2 = cpu_to_le32(glink->features);
418
419 trace_qcom_glink_cmd_version_ack_tx(glink->label, msg.param1, msg.param2);
420
421 qcom_glink_tx(glink, hdr: &msg, hlen: sizeof(msg), NULL, dlen: 0, wait: true);
422}
423
424static void qcom_glink_send_open_ack(struct qcom_glink *glink,
425 struct glink_channel *channel)
426{
427 struct glink_msg msg;
428
429 msg.cmd = cpu_to_le16(GLINK_CMD_OPEN_ACK);
430 msg.param1 = cpu_to_le16(channel->rcid);
431 msg.param2 = cpu_to_le32(0);
432
433 trace_qcom_glink_cmd_open_ack_tx(glink->label, channel->name,
434 channel->lcid, channel->rcid);
435
436 qcom_glink_tx(glink, hdr: &msg, hlen: sizeof(msg), NULL, dlen: 0, wait: true);
437}
438
439static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink,
440 unsigned int cid, bool granted)
441{
442 struct glink_channel *channel;
443 unsigned long flags;
444
445 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
446
447 spin_lock_irqsave(&glink->idr_lock, flags);
448 channel = idr_find(&glink->rcids, id: cid);
449 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
450
451 trace_qcom_glink_cmd_rx_intent_req_ack_rx(glink->label,
452 channel ? channel->name : NULL,
453 channel ? channel->lcid : 0,
454 cid, granted);
455 if (!channel) {
456 dev_err(glink->dev, "unable to find channel\n");
457 return;
458 }
459
460 WRITE_ONCE(channel->intent_req_result, granted);
461 wake_up_all(&channel->intent_req_wq);
462}
463
464static void qcom_glink_intent_req_abort(struct glink_channel *channel)
465{
466 WRITE_ONCE(channel->intent_req_result, 0);
467 wake_up_all(&channel->intent_req_wq);
468}
469
470/**
471 * qcom_glink_send_open_req() - send a GLINK_CMD_OPEN request to the remote
472 * @glink: Ptr to the glink edge
473 * @channel: Ptr to the channel that the open req is sent
474 *
475 * Allocates a local channel id and sends a GLINK_CMD_OPEN message to the remote.
476 * Will return with refcount held, regardless of outcome.
477 *
478 * Return: 0 on success, negative errno otherwise.
479 */
480static int qcom_glink_send_open_req(struct qcom_glink *glink,
481 struct glink_channel *channel)
482{
483 DEFINE_RAW_FLEX(struct glink_msg, req, data, GLINK_NAME_SIZE);
484 int name_len = strlen(channel->name) + 1;
485 int req_len = ALIGN(sizeof(*req) + name_len, 8);
486 int ret;
487 unsigned long flags;
488
489 kref_get(kref: &channel->refcount);
490
491 spin_lock_irqsave(&glink->idr_lock, flags);
492 ret = idr_alloc_cyclic(&glink->lcids, ptr: channel,
493 RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX,
494 GFP_ATOMIC);
495 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
496 if (ret < 0)
497 return ret;
498
499 channel->lcid = ret;
500
501 req->cmd = cpu_to_le16(GLINK_CMD_OPEN);
502 req->param1 = cpu_to_le16(channel->lcid);
503 req->param2 = cpu_to_le32(name_len);
504 strcpy(p: req->data, q: channel->name);
505
506 trace_qcom_glink_cmd_open_tx(glink->label, channel->name,
507 channel->lcid, channel->rcid);
508
509 ret = qcom_glink_tx(glink, hdr: req, hlen: req_len, NULL, dlen: 0, wait: true);
510 if (ret)
511 goto remove_idr;
512
513 return 0;
514
515remove_idr:
516 spin_lock_irqsave(&glink->idr_lock, flags);
517 idr_remove(&glink->lcids, id: channel->lcid);
518 channel->lcid = 0;
519 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
520
521 return ret;
522}
523
524static void qcom_glink_send_close_req(struct qcom_glink *glink,
525 struct glink_channel *channel)
526{
527 struct glink_msg req;
528
529 req.cmd = cpu_to_le16(GLINK_CMD_CLOSE);
530 req.param1 = cpu_to_le16(channel->lcid);
531 req.param2 = 0;
532
533 trace_qcom_glink_cmd_close_tx(glink->label, channel->name,
534 channel->lcid, channel->rcid);
535
536 qcom_glink_tx(glink, hdr: &req, hlen: sizeof(req), NULL, dlen: 0, wait: true);
537}
538
539static void qcom_glink_send_close_ack(struct qcom_glink *glink,
540 struct glink_channel *channel)
541{
542 struct glink_msg req;
543
544 req.cmd = cpu_to_le16(GLINK_CMD_CLOSE_ACK);
545 req.param1 = cpu_to_le16(channel->rcid);
546 req.param2 = 0;
547
548 trace_qcom_glink_cmd_close_ack_tx(glink->label, channel->name,
549 channel->lcid, channel->rcid);
550
551 qcom_glink_tx(glink, hdr: &req, hlen: sizeof(req), NULL, dlen: 0, wait: true);
552}
553
554static void qcom_glink_rx_done_work(struct work_struct *work)
555{
556 struct glink_channel *channel = container_of(work, struct glink_channel,
557 intent_work);
558 struct qcom_glink *glink = channel->glink;
559 struct glink_core_rx_intent *intent, *tmp;
560 struct {
561 u16 id;
562 u16 lcid;
563 u32 liid;
564 } __packed cmd;
565
566 unsigned int cid = channel->lcid;
567 unsigned int iid;
568 bool reuse;
569 unsigned long flags;
570
571 spin_lock_irqsave(&channel->intent_lock, flags);
572 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
573 list_del(entry: &intent->node);
574 spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
575 iid = intent->id;
576 reuse = intent->reuse;
577
578 cmd.id = reuse ? GLINK_CMD_RX_DONE_W_REUSE : GLINK_CMD_RX_DONE;
579 cmd.lcid = cid;
580 cmd.liid = iid;
581
582 trace_qcom_glink_cmd_rx_done_tx(glink->label, channel->name,
583 channel->lcid, channel->rcid, cmd.liid, reuse);
584
585 qcom_glink_tx(glink, hdr: &cmd, hlen: sizeof(cmd), NULL, dlen: 0, wait: true);
586 if (!reuse) {
587 kfree(objp: intent->data);
588 kfree(objp: intent);
589 }
590 spin_lock_irqsave(&channel->intent_lock, flags);
591 }
592 spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
593}
594
595static void qcom_glink_rx_done(struct qcom_glink *glink,
596 struct glink_channel *channel,
597 struct glink_core_rx_intent *intent)
598{
599 /* We don't send RX_DONE to intentless systems */
600 if (glink->intentless) {
601 kfree(objp: intent->data);
602 kfree(objp: intent);
603 return;
604 }
605
606 /* Take it off the tree of receive intents */
607 if (!intent->reuse) {
608 spin_lock(lock: &channel->intent_lock);
609 idr_remove(&channel->liids, id: intent->id);
610 spin_unlock(lock: &channel->intent_lock);
611 }
612
613 /* Schedule the sending of a rx_done indication */
614 spin_lock(lock: &channel->intent_lock);
615 list_add_tail(new: &intent->node, head: &channel->done_intents);
616 spin_unlock(lock: &channel->intent_lock);
617
618 schedule_work(work: &channel->intent_work);
619}
620
621/**
622 * qcom_glink_receive_version() - receive version/features from remote system
623 *
624 * @glink: pointer to transport interface
625 * @version: remote version
626 * @features: remote features
627 *
628 * This function is called in response to a remote-initiated version/feature
629 * negotiation sequence.
630 */
631static void qcom_glink_receive_version(struct qcom_glink *glink,
632 u32 version,
633 u32 features)
634{
635 trace_qcom_glink_cmd_version_rx(glink->label, version, features);
636
637 switch (version) {
638 case 0:
639 break;
640 case GLINK_VERSION_1:
641 glink->features &= features;
642 fallthrough;
643 default:
644 qcom_glink_send_version_ack(glink);
645 break;
646 }
647}
648
649/**
650 * qcom_glink_receive_version_ack() - receive negotiation ack from remote system
651 *
652 * @glink: pointer to transport interface
653 * @version: remote version response
654 * @features: remote features response
655 *
656 * This function is called in response to a local-initiated version/feature
657 * negotiation sequence and is the counter-offer from the remote side based
658 * upon the initial version and feature set requested.
659 */
660static void qcom_glink_receive_version_ack(struct qcom_glink *glink,
661 u32 version,
662 u32 features)
663{
664 trace_qcom_glink_cmd_version_ack_rx(glink->label, version, features);
665
666 switch (version) {
667 case 0:
668 /* Version negotiation failed */
669 break;
670 case GLINK_VERSION_1:
671 if (features == glink->features)
672 break;
673
674 glink->features &= features;
675 fallthrough;
676 default:
677 qcom_glink_send_version(glink);
678 break;
679 }
680}
681
682/**
683 * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to
684 * wire format and transmit
685 * @glink: The transport to transmit on.
686 * @channel: The glink channel
687 * @granted: The request response to encode.
688 *
689 * Return: 0 on success or standard Linux error code.
690 */
691static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink,
692 struct glink_channel *channel,
693 bool granted)
694{
695 struct glink_msg msg;
696
697 trace_qcom_glink_cmd_rx_intent_req_ack_tx(glink->label, channel->name,
698 channel->lcid, channel->rcid,
699 granted);
700
701 msg.cmd = cpu_to_le16(GLINK_CMD_RX_INTENT_REQ_ACK);
702 msg.param1 = cpu_to_le16(channel->lcid);
703 msg.param2 = cpu_to_le32(granted);
704
705 qcom_glink_tx(glink, hdr: &msg, hlen: sizeof(msg), NULL, dlen: 0, wait: true);
706
707 return 0;
708}
709
710/**
711 * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and
712 * transmit
713 * @glink: The transport to transmit on.
714 * @channel: The local channel
715 * @intent: The intent to pass on to remote.
716 *
717 * Return: 0 on success or standard Linux error code.
718 */
719static int qcom_glink_advertise_intent(struct qcom_glink *glink,
720 struct glink_channel *channel,
721 struct glink_core_rx_intent *intent)
722{
723 struct command {
724 __le16 id;
725 __le16 lcid;
726 __le32 count;
727 __le32 size;
728 __le32 liid;
729 } __packed;
730 struct command cmd;
731
732 cmd.id = cpu_to_le16(GLINK_CMD_INTENT);
733 cmd.lcid = cpu_to_le16(channel->lcid);
734 cmd.count = cpu_to_le32(1);
735 cmd.size = cpu_to_le32(intent->size);
736 cmd.liid = cpu_to_le32(intent->id);
737
738 trace_qcom_glink_cmd_intent_tx(glink->label, channel->name,
739 channel->lcid, channel->rcid,
740 cmd.count, cmd.size, cmd.liid);
741
742 qcom_glink_tx(glink, hdr: &cmd, hlen: sizeof(cmd), NULL, dlen: 0, wait: true);
743
744 return 0;
745}
746
747static struct glink_core_rx_intent *
748qcom_glink_alloc_intent(struct qcom_glink *glink,
749 struct glink_channel *channel,
750 size_t size,
751 bool reuseable)
752{
753 struct glink_core_rx_intent *intent;
754 int ret;
755 unsigned long flags;
756
757 intent = kzalloc(sizeof(*intent), GFP_KERNEL);
758 if (!intent)
759 return NULL;
760
761 intent->data = kzalloc(size, GFP_KERNEL);
762 if (!intent->data)
763 goto free_intent;
764
765 spin_lock_irqsave(&channel->intent_lock, flags);
766 ret = idr_alloc_cyclic(&channel->liids, ptr: intent, start: 1, end: -1, GFP_ATOMIC);
767 if (ret < 0) {
768 spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
769 goto free_data;
770 }
771 spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
772
773 intent->id = ret;
774 intent->size = size;
775 intent->reuse = reuseable;
776
777 return intent;
778
779free_data:
780 kfree(objp: intent->data);
781free_intent:
782 kfree(objp: intent);
783 return NULL;
784}
785
786static void qcom_glink_handle_rx_done(struct qcom_glink *glink,
787 u32 cid, uint32_t iid,
788 bool reuse)
789{
790 struct glink_core_rx_intent *intent;
791 struct glink_channel *channel;
792 unsigned long flags;
793
794 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
795
796 spin_lock_irqsave(&glink->idr_lock, flags);
797 channel = idr_find(&glink->rcids, id: cid);
798 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
799
800 trace_qcom_glink_cmd_rx_done_rx(glink->label, channel ? channel->name : NULL,
801 channel ? channel->lcid : 0, cid, iid, reuse);
802 if (!channel) {
803 dev_err(glink->dev, "invalid channel id received\n");
804 return;
805 }
806
807 spin_lock_irqsave(&channel->intent_lock, flags);
808 intent = idr_find(&channel->riids, id: iid);
809
810 if (!intent) {
811 spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
812 dev_err(glink->dev, "invalid intent id received\n");
813 return;
814 }
815
816 intent->in_use = false;
817
818 if (!reuse) {
819 idr_remove(&channel->riids, id: intent->id);
820 kfree(objp: intent);
821 }
822 spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
823
824 if (reuse) {
825 WRITE_ONCE(channel->intent_received, true);
826 wake_up_all(&channel->intent_req_wq);
827 }
828}
829
830/**
831 * qcom_glink_handle_intent_req() - Receive a request for rx_intent
832 * from remote side
833 * @glink: Pointer to the transport interface
834 * @cid: Remote channel ID
835 * @size: size of the intent
836 *
837 * The function searches for the local channel to which the request for
838 * rx_intent has arrived and allocates and notifies the remote back
839 */
840static void qcom_glink_handle_intent_req(struct qcom_glink *glink,
841 u32 cid, size_t size)
842{
843 struct glink_core_rx_intent *intent;
844 struct glink_channel *channel;
845 unsigned long flags;
846
847 spin_lock_irqsave(&glink->idr_lock, flags);
848 channel = idr_find(&glink->rcids, id: cid);
849 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
850
851 trace_qcom_glink_cmd_rx_intent_req_rx(glink->label,
852 channel ? channel->name : NULL,
853 channel ? channel->lcid : 0,
854 cid, size);
855 if (!channel) {
856 pr_err("%s channel not found for cid %d\n", __func__, cid);
857 return;
858 }
859
860 intent = qcom_glink_alloc_intent(glink, channel, size, reuseable: false);
861 if (intent)
862 qcom_glink_advertise_intent(glink, channel, intent);
863
864 qcom_glink_send_intent_req_ack(glink, channel, granted: !!intent);
865}
866
867static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra)
868{
869 struct glink_defer_cmd *dcmd;
870
871 extra = ALIGN(extra, 8);
872
873 if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) {
874 dev_dbg(glink->dev, "Insufficient data in rx fifo");
875 return -ENXIO;
876 }
877
878 dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC);
879 if (!dcmd)
880 return -ENOMEM;
881
882 INIT_LIST_HEAD(list: &dcmd->node);
883
884 qcom_glink_rx_peek(glink,
885 container_of(&dcmd->msg, struct glink_msg, hdr), offset: 0,
886 count: sizeof(dcmd->msg) + extra);
887
888 spin_lock(lock: &glink->rx_lock);
889 list_add_tail(new: &dcmd->node, head: &glink->rx_queue);
890 spin_unlock(lock: &glink->rx_lock);
891
892 schedule_work(work: &glink->rx_work);
893 qcom_glink_rx_advance(glink, count: sizeof(dcmd->msg) + extra);
894
895 return 0;
896}
897
898static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail)
899{
900 struct glink_core_rx_intent *intent;
901 struct glink_channel *channel;
902 struct {
903 struct glink_msg_hdr msg;
904 __le32 chunk_size;
905 __le32 left_size;
906 } __packed hdr;
907 unsigned int chunk_size;
908 unsigned int left_size;
909 unsigned int rcid;
910 unsigned int liid;
911 int ret = 0;
912 unsigned long flags;
913
914 if (avail < sizeof(hdr)) {
915 dev_dbg(glink->dev, "Not enough data in fifo\n");
916 return -EAGAIN;
917 }
918
919 qcom_glink_rx_peek(glink, data: &hdr, offset: 0, count: sizeof(hdr));
920 chunk_size = le32_to_cpu(hdr.chunk_size);
921 left_size = le32_to_cpu(hdr.left_size);
922
923 if (avail < sizeof(hdr) + chunk_size) {
924 dev_dbg(glink->dev, "Payload not yet in fifo\n");
925 return -EAGAIN;
926 }
927
928 rcid = le16_to_cpu(hdr.msg.param1);
929 liid = le32_to_cpu(hdr.msg.param2);
930 spin_lock_irqsave(&glink->idr_lock, flags);
931 channel = idr_find(&glink->rcids, id: rcid);
932 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
933
934 trace_qcom_glink_cmd_tx_data_rx(glink->label, channel ? channel->name : NULL,
935 channel ? channel->lcid : 0, rcid,
936 liid, chunk_size, left_size,
937 hdr.msg.cmd == GLINK_CMD_TX_DATA_CONT);
938 if (!channel) {
939 dev_dbg(glink->dev, "Data on non-existing channel\n");
940
941 /* Drop the message */
942 goto advance_rx;
943 }
944
945 if (glink->intentless) {
946 /* Might have an ongoing, fragmented, message to append */
947 if (!channel->buf) {
948 intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
949 if (!intent)
950 return -ENOMEM;
951
952 intent->data = kmalloc(chunk_size + left_size,
953 GFP_ATOMIC);
954 if (!intent->data) {
955 kfree(objp: intent);
956 return -ENOMEM;
957 }
958
959 intent->id = 0xbabababa;
960 intent->size = chunk_size + left_size;
961 intent->offset = 0;
962
963 channel->buf = intent;
964 } else {
965 intent = channel->buf;
966 }
967 } else {
968 spin_lock_irqsave(&channel->intent_lock, flags);
969 intent = idr_find(&channel->liids, id: liid);
970 spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
971
972 if (!intent) {
973 dev_err(glink->dev,
974 "no intent found for channel %s intent %d",
975 channel->name, liid);
976 ret = -ENOENT;
977 goto advance_rx;
978 }
979 }
980
981 if (intent->size - intent->offset < chunk_size) {
982 dev_err(glink->dev, "Insufficient space in intent\n");
983
984 /* The packet header lied, drop payload */
985 goto advance_rx;
986 }
987
988 qcom_glink_rx_peek(glink, data: intent->data + intent->offset,
989 offset: sizeof(hdr), count: chunk_size);
990 intent->offset += chunk_size;
991
992 /* Handle message when no fragments remain to be received */
993 if (!left_size) {
994 spin_lock(lock: &channel->recv_lock);
995 if (channel->ept.cb) {
996 channel->ept.cb(channel->ept.rpdev,
997 intent->data,
998 intent->offset,
999 channel->ept.priv,
1000 RPMSG_ADDR_ANY);
1001 }
1002 spin_unlock(lock: &channel->recv_lock);
1003
1004 intent->offset = 0;
1005 channel->buf = NULL;
1006
1007 qcom_glink_rx_done(glink, channel, intent);
1008 }
1009
1010advance_rx:
1011 qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8));
1012
1013 return ret;
1014}
1015
1016static void qcom_glink_rx_read_notif(struct qcom_glink *glink)
1017{
1018 trace_qcom_glink_cmd_read_notif_rx(glink->label);
1019
1020 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
1021 qcom_glink_tx_kick(glink);
1022}
1023
1024static void qcom_glink_handle_intent(struct qcom_glink *glink,
1025 unsigned int cid,
1026 unsigned int count,
1027 size_t avail)
1028{
1029 struct glink_core_rx_intent *intent;
1030 struct glink_channel *channel;
1031 struct intent_pair {
1032 __le32 size;
1033 __le32 iid;
1034 };
1035
1036 struct {
1037 struct glink_msg_hdr msg;
1038 struct intent_pair intents[];
1039 } __packed * msg;
1040
1041 const size_t msglen = struct_size(msg, intents, count);
1042 int ret;
1043 int i;
1044 unsigned long flags;
1045
1046 if (avail < msglen) {
1047 dev_dbg(glink->dev, "Not enough data in fifo\n");
1048 return;
1049 }
1050
1051 spin_lock_irqsave(&glink->idr_lock, flags);
1052 channel = idr_find(&glink->rcids, id: cid);
1053 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1054 if (!channel) {
1055 trace_qcom_glink_cmd_intent_rx(glink->label, NULL, 0, cid, count, 0, 0);
1056 dev_err(glink->dev, "intents for non-existing channel\n");
1057 qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
1058 return;
1059 }
1060
1061 msg = kmalloc(msglen, GFP_ATOMIC);
1062 if (!msg)
1063 return;
1064
1065 qcom_glink_rx_peek(glink, data: msg, offset: 0, count: msglen);
1066
1067 trace_qcom_glink_cmd_intent_rx(glink->label, channel->name,
1068 channel->lcid, cid, count,
1069 count > 0 ? msg->intents[0].size : 0,
1070 count > 0 ? msg->intents[0].iid : 0);
1071
1072 for (i = 0; i < count; ++i) {
1073 intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
1074 if (!intent)
1075 break;
1076
1077 intent->id = le32_to_cpu(msg->intents[i].iid);
1078 intent->size = le32_to_cpu(msg->intents[i].size);
1079
1080 spin_lock_irqsave(&channel->intent_lock, flags);
1081 ret = idr_alloc(&channel->riids, ptr: intent,
1082 start: intent->id, end: intent->id + 1, GFP_ATOMIC);
1083 spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
1084
1085 if (ret < 0)
1086 dev_err(glink->dev, "failed to store remote intent\n");
1087 }
1088
1089 WRITE_ONCE(channel->intent_received, true);
1090 wake_up_all(&channel->intent_req_wq);
1091
1092 kfree(objp: msg);
1093 qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
1094}
1095
1096static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid)
1097{
1098 struct glink_channel *channel;
1099
1100 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
1101
1102 spin_lock(lock: &glink->idr_lock);
1103 channel = idr_find(&glink->lcids, id: lcid);
1104 spin_unlock(lock: &glink->idr_lock);
1105
1106 trace_qcom_glink_cmd_open_ack_rx(glink->label, channel ? channel->name : NULL,
1107 lcid, channel ? channel->rcid : 0);
1108 if (!channel) {
1109 dev_err(glink->dev, "Invalid open ack packet\n");
1110 return -EINVAL;
1111 }
1112
1113 complete_all(&channel->open_ack);
1114
1115 return 0;
1116}
1117
1118/**
1119 * qcom_glink_set_flow_control() - convert a signal cmd to wire format and transmit
1120 * @ept: Rpmsg endpoint for channel.
1121 * @pause: Pause transmission
1122 * @dst: destination address of the endpoint
1123 *
1124 * Return: 0 on success or standard Linux error code.
1125 */
1126static int qcom_glink_set_flow_control(struct rpmsg_endpoint *ept, bool pause, u32 dst)
1127{
1128 struct glink_channel *channel = to_glink_channel(ept);
1129 struct qcom_glink *glink = channel->glink;
1130 struct glink_msg msg;
1131 u32 sigs = 0;
1132
1133 if (pause)
1134 sigs |= NATIVE_DTR_SIG | NATIVE_RTS_SIG;
1135
1136 msg.cmd = cpu_to_le16(GLINK_CMD_SIGNALS);
1137 msg.param1 = cpu_to_le16(channel->lcid);
1138 msg.param2 = cpu_to_le32(sigs);
1139
1140 trace_qcom_glink_cmd_signal_tx(glink->label, channel->name,
1141 channel->lcid, channel->rcid, sigs);
1142
1143 return qcom_glink_tx(glink, hdr: &msg, hlen: sizeof(msg), NULL, dlen: 0, wait: true);
1144}
1145
1146static void qcom_glink_handle_signals(struct qcom_glink *glink,
1147 unsigned int rcid, unsigned int sigs)
1148{
1149 struct glink_channel *channel;
1150 unsigned long flags;
1151 bool enable;
1152
1153 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
1154
1155 spin_lock_irqsave(&glink->idr_lock, flags);
1156 channel = idr_find(&glink->rcids, id: rcid);
1157 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1158
1159 trace_qcom_glink_cmd_signal_rx(glink->label, channel ? channel->name : NULL,
1160 channel ? channel->lcid : 0, rcid, sigs);
1161 if (!channel) {
1162 dev_err(glink->dev, "signal for non-existing channel\n");
1163 return;
1164 }
1165
1166 enable = sigs & NATIVE_DSR_SIG || sigs & NATIVE_CTS_SIG;
1167
1168 if (channel->ept.flow_cb)
1169 channel->ept.flow_cb(channel->ept.rpdev, channel->ept.priv, enable);
1170}
1171
1172void qcom_glink_native_rx(struct qcom_glink *glink)
1173{
1174 struct glink_msg msg;
1175 unsigned int param1;
1176 unsigned int param2;
1177 unsigned int avail;
1178 unsigned int cmd;
1179 int ret = 0;
1180
1181 /* To wakeup any blocking writers */
1182 wake_up_all(&glink->tx_avail_notify);
1183
1184 for (;;) {
1185 avail = qcom_glink_rx_avail(glink);
1186 if (avail < sizeof(msg))
1187 break;
1188
1189 qcom_glink_rx_peek(glink, data: &msg, offset: 0, count: sizeof(msg));
1190
1191 cmd = le16_to_cpu(msg.cmd);
1192 param1 = le16_to_cpu(msg.param1);
1193 param2 = le32_to_cpu(msg.param2);
1194
1195 switch (cmd) {
1196 case GLINK_CMD_VERSION:
1197 case GLINK_CMD_VERSION_ACK:
1198 case GLINK_CMD_CLOSE:
1199 case GLINK_CMD_CLOSE_ACK:
1200 case GLINK_CMD_RX_INTENT_REQ:
1201 ret = qcom_glink_rx_defer(glink, extra: 0);
1202 break;
1203 case GLINK_CMD_OPEN_ACK:
1204 ret = qcom_glink_rx_open_ack(glink, lcid: param1);
1205 break;
1206 case GLINK_CMD_OPEN:
1207 /* upper 16 bits of param2 are the "prio" field */
1208 ret = qcom_glink_rx_defer(glink, extra: param2 & 0xffff);
1209 break;
1210 case GLINK_CMD_TX_DATA:
1211 case GLINK_CMD_TX_DATA_CONT:
1212 ret = qcom_glink_rx_data(glink, avail);
1213 break;
1214 case GLINK_CMD_READ_NOTIF:
1215 qcom_glink_rx_read_notif(glink);
1216 break;
1217 case GLINK_CMD_INTENT:
1218 qcom_glink_handle_intent(glink, cid: param1, count: param2, avail);
1219 break;
1220 case GLINK_CMD_RX_DONE:
1221 qcom_glink_handle_rx_done(glink, cid: param1, iid: param2, reuse: false);
1222 break;
1223 case GLINK_CMD_RX_DONE_W_REUSE:
1224 qcom_glink_handle_rx_done(glink, cid: param1, iid: param2, reuse: true);
1225 break;
1226 case GLINK_CMD_RX_INTENT_REQ_ACK:
1227 qcom_glink_handle_intent_req_ack(glink, cid: param1, granted: param2);
1228 break;
1229 case GLINK_CMD_SIGNALS:
1230 qcom_glink_handle_signals(glink, rcid: param1, sigs: param2);
1231 break;
1232 default:
1233 dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd);
1234 ret = -EINVAL;
1235 break;
1236 }
1237
1238 if (ret)
1239 break;
1240 }
1241}
1242EXPORT_SYMBOL(qcom_glink_native_rx);
1243
1244/* Locally initiated rpmsg_create_ept */
1245static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink,
1246 const char *name)
1247{
1248 struct glink_channel *channel;
1249 int ret;
1250 unsigned long flags;
1251
1252 channel = qcom_glink_alloc_channel(glink, name);
1253 if (IS_ERR(ptr: channel))
1254 return ERR_CAST(ptr: channel);
1255
1256 ret = qcom_glink_send_open_req(glink, channel);
1257 if (ret)
1258 goto release_channel;
1259
1260 ret = wait_for_completion_timeout(x: &channel->open_ack, timeout: 5 * HZ);
1261 if (!ret)
1262 goto err_timeout;
1263
1264 ret = wait_for_completion_timeout(x: &channel->open_req, timeout: 5 * HZ);
1265 if (!ret)
1266 goto err_timeout;
1267
1268 qcom_glink_send_open_ack(glink, channel);
1269
1270 return channel;
1271
1272err_timeout:
1273 /* qcom_glink_send_open_req() did register the channel in lcids*/
1274 spin_lock_irqsave(&glink->idr_lock, flags);
1275 idr_remove(&glink->lcids, id: channel->lcid);
1276 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1277
1278release_channel:
1279 /* Release qcom_glink_send_open_req() reference */
1280 kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1281 /* Release qcom_glink_alloc_channel() reference */
1282 kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1283
1284 return ERR_PTR(error: -ETIMEDOUT);
1285}
1286
1287/* Remote initiated rpmsg_create_ept */
1288static int qcom_glink_create_remote(struct qcom_glink *glink,
1289 struct glink_channel *channel)
1290{
1291 int ret;
1292
1293 qcom_glink_send_open_ack(glink, channel);
1294
1295 ret = qcom_glink_send_open_req(glink, channel);
1296 if (ret)
1297 goto close_link;
1298
1299 ret = wait_for_completion_timeout(x: &channel->open_ack, timeout: 5 * HZ);
1300 if (!ret) {
1301 ret = -ETIMEDOUT;
1302 goto close_link;
1303 }
1304
1305 return 0;
1306
1307close_link:
1308 /*
1309 * Send a close request to "undo" our open-ack. The close-ack will
1310 * release qcom_glink_send_open_req() reference and the last reference
1311 * will be relesed after receiving remote_close or transport unregister
1312 * by calling qcom_glink_native_remove().
1313 */
1314 qcom_glink_send_close_req(glink, channel);
1315
1316 return ret;
1317}
1318
1319static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev,
1320 rpmsg_rx_cb_t cb,
1321 void *priv,
1322 struct rpmsg_channel_info
1323 chinfo)
1324{
1325 struct glink_channel *parent = to_glink_channel(rpdev->ept);
1326 struct glink_channel *channel;
1327 struct qcom_glink *glink = parent->glink;
1328 struct rpmsg_endpoint *ept;
1329 const char *name = chinfo.name;
1330 int cid;
1331 int ret;
1332 unsigned long flags;
1333
1334 spin_lock_irqsave(&glink->idr_lock, flags);
1335 idr_for_each_entry(&glink->rcids, channel, cid) {
1336 if (!strcmp(channel->name, name))
1337 break;
1338 }
1339 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1340
1341 if (!channel) {
1342 channel = qcom_glink_create_local(glink, name);
1343 if (IS_ERR(ptr: channel))
1344 return NULL;
1345 } else {
1346 ret = qcom_glink_create_remote(glink, channel);
1347 if (ret)
1348 return NULL;
1349 }
1350
1351 ept = &channel->ept;
1352 ept->rpdev = rpdev;
1353 ept->cb = cb;
1354 ept->priv = priv;
1355 ept->ops = &glink_endpoint_ops;
1356
1357 return ept;
1358}
1359
1360static int qcom_glink_announce_create(struct rpmsg_device *rpdev)
1361{
1362 struct glink_channel *channel = to_glink_channel(rpdev->ept);
1363 struct device_node *np = rpdev->dev.of_node;
1364 struct qcom_glink *glink = channel->glink;
1365 struct glink_core_rx_intent *intent;
1366 const struct property *prop = NULL;
1367 __be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) };
1368 int num_intents;
1369 int num_groups = 1;
1370 __be32 *val = defaults;
1371 int size;
1372
1373 if (glink->intentless || !completion_done(x: &channel->open_ack))
1374 return 0;
1375
1376 prop = of_find_property(np, name: "qcom,intents", NULL);
1377 if (prop) {
1378 val = prop->value;
1379 num_groups = prop->length / sizeof(u32) / 2;
1380 }
1381
1382 /* Channel is now open, advertise base set of intents */
1383 while (num_groups--) {
1384 size = be32_to_cpup(p: val++);
1385 num_intents = be32_to_cpup(p: val++);
1386 while (num_intents--) {
1387 intent = qcom_glink_alloc_intent(glink, channel, size,
1388 reuseable: true);
1389 if (!intent)
1390 break;
1391
1392 qcom_glink_advertise_intent(glink, channel, intent);
1393 }
1394 }
1395 return 0;
1396}
1397
1398static void qcom_glink_remove_rpmsg_device(struct qcom_glink *glink, struct glink_channel *channel)
1399{
1400 struct rpmsg_channel_info chinfo;
1401
1402 if (channel->rpdev) {
1403 strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name));
1404 chinfo.src = RPMSG_ADDR_ANY;
1405 chinfo.dst = RPMSG_ADDR_ANY;
1406 rpmsg_unregister_device(parent: glink->dev, chinfo: &chinfo);
1407 }
1408 channel->rpdev = NULL;
1409}
1410
1411static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept)
1412{
1413 struct glink_channel *channel = to_glink_channel(ept);
1414 struct qcom_glink *glink = channel->glink;
1415 unsigned long flags;
1416
1417 spin_lock_irqsave(&channel->recv_lock, flags);
1418 channel->ept.cb = NULL;
1419 spin_unlock_irqrestore(lock: &channel->recv_lock, flags);
1420
1421 /* Decouple the potential rpdev from the channel */
1422 qcom_glink_remove_rpmsg_device(glink, channel);
1423
1424 qcom_glink_send_close_req(glink, channel);
1425}
1426
1427static int qcom_glink_request_intent(struct qcom_glink *glink,
1428 struct glink_channel *channel,
1429 size_t size)
1430{
1431 struct {
1432 u16 id;
1433 u16 cid;
1434 u32 size;
1435 } __packed cmd;
1436
1437 int ret;
1438
1439 mutex_lock(&channel->intent_req_lock);
1440
1441 WRITE_ONCE(channel->intent_req_result, -1);
1442 WRITE_ONCE(channel->intent_received, false);
1443
1444 cmd.id = GLINK_CMD_RX_INTENT_REQ;
1445 cmd.cid = channel->lcid;
1446 cmd.size = size;
1447
1448 trace_qcom_glink_cmd_rx_intent_req_tx(glink->label, channel->name,
1449 channel->lcid, channel->rcid,
1450 cmd.size);
1451
1452 ret = qcom_glink_tx(glink, hdr: &cmd, hlen: sizeof(cmd), NULL, dlen: 0, wait: true);
1453 if (ret)
1454 goto unlock;
1455
1456 ret = wait_event_timeout(channel->intent_req_wq,
1457 READ_ONCE(channel->intent_req_result) == 0 ||
1458 (READ_ONCE(channel->intent_req_result) > 0 &&
1459 READ_ONCE(channel->intent_received)) ||
1460 glink->abort_tx,
1461 10 * HZ);
1462 if (!ret) {
1463 dev_err(glink->dev, "intent request timed out\n");
1464 ret = -ETIMEDOUT;
1465 } else if (glink->abort_tx) {
1466 ret = -ECANCELED;
1467 } else {
1468 ret = READ_ONCE(channel->intent_req_result) ? 0 : -EAGAIN;
1469 }
1470
1471unlock:
1472 mutex_unlock(lock: &channel->intent_req_lock);
1473 return ret;
1474}
1475
1476static int __qcom_glink_send(struct glink_channel *channel,
1477 void *data, int len, bool wait)
1478{
1479 struct qcom_glink *glink = channel->glink;
1480 struct glink_core_rx_intent *intent = NULL;
1481 struct glink_core_rx_intent *tmp;
1482 int iid = 0;
1483 struct {
1484 struct glink_msg_hdr msg;
1485 __le32 chunk_size;
1486 __le32 left_size;
1487 } __packed req;
1488 int ret;
1489 unsigned long flags;
1490 int chunk_size = len;
1491 size_t offset = 0;
1492
1493 if (!glink->intentless) {
1494 while (!intent) {
1495 spin_lock_irqsave(&channel->intent_lock, flags);
1496 idr_for_each_entry(&channel->riids, tmp, iid) {
1497 if (tmp->size >= len && !tmp->in_use) {
1498 if (!intent)
1499 intent = tmp;
1500 else if (intent->size > tmp->size)
1501 intent = tmp;
1502 if (intent->size == len)
1503 break;
1504 }
1505 }
1506 if (intent)
1507 intent->in_use = true;
1508 spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
1509
1510 /* We found an available intent */
1511 if (intent)
1512 break;
1513
1514 if (!wait)
1515 return -EBUSY;
1516
1517 ret = qcom_glink_request_intent(glink, channel, size: len);
1518 if (ret < 0)
1519 return ret;
1520 }
1521
1522 iid = intent->id;
1523 }
1524
1525 while (offset < len) {
1526 chunk_size = len - offset;
1527 if (chunk_size > SZ_8K && wait)
1528 chunk_size = SZ_8K;
1529
1530 req.msg.cmd = cpu_to_le16(offset == 0 ? GLINK_CMD_TX_DATA : GLINK_CMD_TX_DATA_CONT);
1531 req.msg.param1 = cpu_to_le16(channel->lcid);
1532 req.msg.param2 = cpu_to_le32(iid);
1533 req.chunk_size = cpu_to_le32(chunk_size);
1534 req.left_size = cpu_to_le32(len - offset - chunk_size);
1535
1536 trace_qcom_glink_cmd_tx_data_tx(glink->label, channel->name,
1537 channel->lcid, channel->rcid,
1538 iid, chunk_size,
1539 len - offset - chunk_size,
1540 offset > 0);
1541
1542 ret = qcom_glink_tx(glink, hdr: &req, hlen: sizeof(req), data: data + offset, dlen: chunk_size, wait);
1543 if (ret) {
1544 /* Mark intent available if we failed */
1545 if (intent)
1546 intent->in_use = false;
1547 return ret;
1548 }
1549
1550 offset += chunk_size;
1551 }
1552
1553 return 0;
1554}
1555
1556static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len)
1557{
1558 struct glink_channel *channel = to_glink_channel(ept);
1559
1560 return __qcom_glink_send(channel, data, len, wait: true);
1561}
1562
1563static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len)
1564{
1565 struct glink_channel *channel = to_glink_channel(ept);
1566
1567 return __qcom_glink_send(channel, data, len, wait: false);
1568}
1569
1570static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1571{
1572 struct glink_channel *channel = to_glink_channel(ept);
1573
1574 return __qcom_glink_send(channel, data, len, wait: true);
1575}
1576
1577static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1578{
1579 struct glink_channel *channel = to_glink_channel(ept);
1580
1581 return __qcom_glink_send(channel, data, len, wait: false);
1582}
1583
1584/*
1585 * Finds the device_node for the glink child interested in this channel.
1586 */
1587static struct device_node *qcom_glink_match_channel(struct device_node *node,
1588 const char *channel)
1589{
1590 struct device_node *child;
1591 const char *name;
1592 const char *key;
1593 int ret;
1594
1595 for_each_available_child_of_node(node, child) {
1596 key = "qcom,glink-channels";
1597 ret = of_property_read_string(np: child, propname: key, out_string: &name);
1598 if (ret)
1599 continue;
1600
1601 if (strcmp(name, channel) == 0)
1602 return child;
1603 }
1604
1605 return NULL;
1606}
1607
1608static const struct rpmsg_device_ops glink_device_ops = {
1609 .create_ept = qcom_glink_create_ept,
1610 .announce_create = qcom_glink_announce_create,
1611};
1612
1613static const struct rpmsg_endpoint_ops glink_endpoint_ops = {
1614 .destroy_ept = qcom_glink_destroy_ept,
1615 .send = qcom_glink_send,
1616 .sendto = qcom_glink_sendto,
1617 .trysend = qcom_glink_trysend,
1618 .trysendto = qcom_glink_trysendto,
1619 .set_flow_control = qcom_glink_set_flow_control,
1620};
1621
1622static void qcom_glink_rpdev_release(struct device *dev)
1623{
1624 struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1625
1626 kfree(objp: rpdev->driver_override);
1627 kfree(objp: rpdev);
1628}
1629
1630static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid,
1631 char *name)
1632{
1633 struct glink_channel *channel;
1634 struct rpmsg_device *rpdev;
1635 bool create_device = false;
1636 struct device_node *node;
1637 int lcid;
1638 int ret;
1639 unsigned long flags;
1640
1641 spin_lock_irqsave(&glink->idr_lock, flags);
1642 idr_for_each_entry(&glink->lcids, channel, lcid) {
1643 if (!strcmp(channel->name, name))
1644 break;
1645 }
1646 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1647
1648 if (!channel) {
1649 channel = qcom_glink_alloc_channel(glink, name);
1650 if (IS_ERR(ptr: channel))
1651 return PTR_ERR(ptr: channel);
1652
1653 /* The opening dance was initiated by the remote */
1654 create_device = true;
1655 }
1656
1657 trace_qcom_glink_cmd_open_rx(glink->label, name, channel->lcid, rcid);
1658
1659 spin_lock_irqsave(&glink->idr_lock, flags);
1660 ret = idr_alloc(&glink->rcids, ptr: channel, start: rcid, end: rcid + 1, GFP_ATOMIC);
1661 if (ret < 0) {
1662 dev_err(glink->dev, "Unable to insert channel into rcid list\n");
1663 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1664 goto free_channel;
1665 }
1666 channel->rcid = ret;
1667 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1668
1669 complete_all(&channel->open_req);
1670
1671 if (create_device) {
1672 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1673 if (!rpdev) {
1674 ret = -ENOMEM;
1675 goto rcid_remove;
1676 }
1677
1678 rpdev->ept = &channel->ept;
1679 strscpy(rpdev->id.name, name);
1680 rpdev->src = RPMSG_ADDR_ANY;
1681 rpdev->dst = RPMSG_ADDR_ANY;
1682 rpdev->ops = &glink_device_ops;
1683
1684 node = qcom_glink_match_channel(node: glink->dev->of_node, channel: name);
1685 rpdev->dev.of_node = node;
1686 rpdev->dev.parent = glink->dev;
1687 rpdev->dev.release = qcom_glink_rpdev_release;
1688
1689 ret = rpmsg_register_device(rpdev);
1690 if (ret)
1691 goto rcid_remove;
1692
1693 channel->rpdev = rpdev;
1694 }
1695
1696 return 0;
1697
1698rcid_remove:
1699 spin_lock_irqsave(&glink->idr_lock, flags);
1700 idr_remove(&glink->rcids, id: channel->rcid);
1701 channel->rcid = 0;
1702 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1703free_channel:
1704 /* Release the reference, iff we took it */
1705 if (create_device)
1706 kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1707
1708 return ret;
1709}
1710
1711static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid)
1712{
1713 struct glink_channel *channel;
1714 unsigned long flags;
1715
1716 spin_lock_irqsave(&glink->idr_lock, flags);
1717 channel = idr_find(&glink->rcids, id: rcid);
1718 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1719
1720 trace_qcom_glink_cmd_close_rx(glink->label, channel ? channel->name : NULL,
1721 channel ? channel->lcid : 0, rcid);
1722 if (WARN(!channel, "close request on unknown channel\n"))
1723 return;
1724
1725 /* cancel pending rx_done work */
1726 cancel_work_sync(work: &channel->intent_work);
1727
1728 qcom_glink_remove_rpmsg_device(glink, channel);
1729
1730 qcom_glink_send_close_ack(glink, channel);
1731
1732 spin_lock_irqsave(&glink->idr_lock, flags);
1733 idr_remove(&glink->rcids, id: channel->rcid);
1734 channel->rcid = 0;
1735 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1736
1737 kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1738}
1739
1740static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid)
1741{
1742 struct glink_channel *channel;
1743 unsigned long flags;
1744
1745 /* To wakeup any blocking writers */
1746 wake_up_all(&glink->tx_avail_notify);
1747
1748 spin_lock_irqsave(&glink->idr_lock, flags);
1749 channel = idr_find(&glink->lcids, id: lcid);
1750
1751 trace_qcom_glink_cmd_close_ack_rx(glink->label, channel ? channel->name : NULL,
1752 lcid, channel ? channel->rcid : 0);
1753 if (WARN(!channel, "close ack on unknown channel\n")) {
1754 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1755 return;
1756 }
1757
1758 idr_remove(&glink->lcids, id: channel->lcid);
1759 channel->lcid = 0;
1760 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1761
1762 /* Decouple the potential rpdev from the channel */
1763 qcom_glink_remove_rpmsg_device(glink, channel);
1764
1765 kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1766}
1767
1768static void qcom_glink_work(struct work_struct *work)
1769{
1770 struct qcom_glink *glink = container_of(work, struct qcom_glink,
1771 rx_work);
1772 struct glink_defer_cmd *dcmd;
1773 struct glink_msg *msg;
1774 unsigned long flags;
1775 unsigned int param1;
1776 unsigned int param2;
1777 unsigned int cmd;
1778
1779 for (;;) {
1780 spin_lock_irqsave(&glink->rx_lock, flags);
1781 if (list_empty(head: &glink->rx_queue)) {
1782 spin_unlock_irqrestore(lock: &glink->rx_lock, flags);
1783 break;
1784 }
1785 dcmd = list_first_entry(&glink->rx_queue,
1786 struct glink_defer_cmd, node);
1787 list_del(entry: &dcmd->node);
1788 spin_unlock_irqrestore(lock: &glink->rx_lock, flags);
1789
1790 msg = container_of(&dcmd->msg, struct glink_msg, hdr);
1791 cmd = le16_to_cpu(msg->cmd);
1792 param1 = le16_to_cpu(msg->param1);
1793 param2 = le32_to_cpu(msg->param2);
1794
1795 switch (cmd) {
1796 case GLINK_CMD_VERSION:
1797 qcom_glink_receive_version(glink, version: param1, features: param2);
1798 break;
1799 case GLINK_CMD_VERSION_ACK:
1800 qcom_glink_receive_version_ack(glink, version: param1, features: param2);
1801 break;
1802 case GLINK_CMD_OPEN:
1803 qcom_glink_rx_open(glink, rcid: param1, name: msg->data);
1804 break;
1805 case GLINK_CMD_CLOSE:
1806 qcom_glink_rx_close(glink, rcid: param1);
1807 break;
1808 case GLINK_CMD_CLOSE_ACK:
1809 qcom_glink_rx_close_ack(glink, lcid: param1);
1810 break;
1811 case GLINK_CMD_RX_INTENT_REQ:
1812 qcom_glink_handle_intent_req(glink, cid: param1, size: param2);
1813 break;
1814 default:
1815 WARN(1, "Unknown defer object %d\n", cmd);
1816 break;
1817 }
1818
1819 kfree(objp: dcmd);
1820 }
1821}
1822
1823static void qcom_glink_cancel_rx_work(struct qcom_glink *glink)
1824{
1825 struct glink_defer_cmd *dcmd;
1826 struct glink_defer_cmd *tmp;
1827
1828 /* cancel any pending deferred rx_work */
1829 cancel_work_sync(work: &glink->rx_work);
1830
1831 list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node)
1832 kfree(objp: dcmd);
1833}
1834
1835static ssize_t rpmsg_name_show(struct device *dev,
1836 struct device_attribute *attr, char *buf)
1837{
1838 int ret = 0;
1839 const char *name;
1840
1841 ret = of_property_read_string(np: dev->of_node, propname: "label", out_string: &name);
1842 if (ret < 0)
1843 name = dev->of_node->name;
1844
1845 return sysfs_emit(buf, fmt: "%s\n", name);
1846}
1847static DEVICE_ATTR_RO(rpmsg_name);
1848
1849static struct attribute *qcom_glink_attrs[] = {
1850 &dev_attr_rpmsg_name.attr,
1851 NULL
1852};
1853ATTRIBUTE_GROUPS(qcom_glink);
1854
1855static void qcom_glink_device_release(struct device *dev)
1856{
1857 struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1858 struct glink_channel *channel = to_glink_channel(rpdev->ept);
1859
1860 /* Release qcom_glink_alloc_channel() reference */
1861 kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1862 kfree(objp: rpdev->driver_override);
1863 kfree(objp: rpdev);
1864}
1865
1866static int qcom_glink_create_chrdev(struct qcom_glink *glink)
1867{
1868 struct rpmsg_device *rpdev;
1869 struct glink_channel *channel;
1870
1871 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1872 if (!rpdev)
1873 return -ENOMEM;
1874
1875 channel = qcom_glink_alloc_channel(glink, name: "rpmsg_chrdev");
1876 if (IS_ERR(ptr: channel)) {
1877 kfree(objp: rpdev);
1878 return PTR_ERR(ptr: channel);
1879 }
1880 channel->rpdev = rpdev;
1881
1882 rpdev->ept = &channel->ept;
1883 rpdev->ops = &glink_device_ops;
1884 rpdev->dev.parent = glink->dev;
1885 rpdev->dev.release = qcom_glink_device_release;
1886
1887 return rpmsg_ctrldev_register_device(rpdev);
1888}
1889
1890struct qcom_glink *qcom_glink_native_probe(struct device *dev,
1891 unsigned long features,
1892 struct qcom_glink_pipe *rx,
1893 struct qcom_glink_pipe *tx,
1894 bool intentless)
1895{
1896 int ret;
1897 struct qcom_glink *glink;
1898
1899 glink = devm_kzalloc(dev, size: sizeof(*glink), GFP_KERNEL);
1900 if (!glink)
1901 return ERR_PTR(error: -ENOMEM);
1902
1903 glink->dev = dev;
1904 glink->tx_pipe = tx;
1905 glink->rx_pipe = rx;
1906
1907 glink->features = features;
1908 glink->intentless = intentless;
1909
1910 spin_lock_init(&glink->tx_lock);
1911 spin_lock_init(&glink->rx_lock);
1912 INIT_LIST_HEAD(list: &glink->rx_queue);
1913 INIT_WORK(&glink->rx_work, qcom_glink_work);
1914 init_waitqueue_head(&glink->tx_avail_notify);
1915
1916 spin_lock_init(&glink->idr_lock);
1917 idr_init(idr: &glink->lcids);
1918 idr_init(idr: &glink->rcids);
1919
1920 ret = of_property_read_string(np: dev->of_node, propname: "label", out_string: &glink->label);
1921 if (ret < 0)
1922 glink->label = dev->of_node->name;
1923
1924 glink->dev->groups = qcom_glink_groups;
1925
1926 ret = device_add_groups(dev, groups: qcom_glink_groups);
1927 if (ret)
1928 dev_err(dev, "failed to add groups\n");
1929
1930 ret = qcom_glink_send_version(glink);
1931 if (ret)
1932 return ERR_PTR(error: ret);
1933
1934 ret = qcom_glink_create_chrdev(glink);
1935 if (ret)
1936 dev_err(glink->dev, "failed to register chrdev\n");
1937
1938 return glink;
1939}
1940EXPORT_SYMBOL_GPL(qcom_glink_native_probe);
1941
1942static int qcom_glink_remove_device(struct device *dev, void *data)
1943{
1944 device_unregister(dev);
1945
1946 return 0;
1947}
1948
1949void qcom_glink_native_remove(struct qcom_glink *glink)
1950{
1951 struct glink_channel *channel;
1952 unsigned long flags;
1953 int cid;
1954 int ret;
1955
1956 qcom_glink_cancel_rx_work(glink);
1957
1958 /* Fail all attempts at sending messages */
1959 spin_lock_irqsave(&glink->tx_lock, flags);
1960 glink->abort_tx = true;
1961 wake_up_all(&glink->tx_avail_notify);
1962 spin_unlock_irqrestore(lock: &glink->tx_lock, flags);
1963
1964 /* Abort any senders waiting for intent requests */
1965 spin_lock_irqsave(&glink->idr_lock, flags);
1966 idr_for_each_entry(&glink->lcids, channel, cid)
1967 qcom_glink_intent_req_abort(channel);
1968 spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1969
1970 ret = device_for_each_child(parent: glink->dev, NULL, fn: qcom_glink_remove_device);
1971 if (ret)
1972 dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret);
1973
1974 /* Release any defunct local channels, waiting for close-ack */
1975 idr_for_each_entry(&glink->lcids, channel, cid)
1976 kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1977
1978 /* Release any defunct local channels, waiting for close-req */
1979 idr_for_each_entry(&glink->rcids, channel, cid)
1980 kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1981
1982 idr_destroy(&glink->lcids);
1983 idr_destroy(&glink->rcids);
1984}
1985EXPORT_SYMBOL_GPL(qcom_glink_native_remove);
1986
1987MODULE_DESCRIPTION("Qualcomm GLINK driver");
1988MODULE_LICENSE("GPL v2");
1989

source code of linux/drivers/rpmsg/qcom_glink_native.c