1	// SPDX-License-Identifier: GPL-2.0-only
2	/* The industrial I/O core
3	*
4	* Copyright (c) 2008 Jonathan Cameron
5	*
6	* Handling of buffer allocation / resizing.
7	*
8	* Things to look at here.
9	* - Better memory allocation techniques?
10	* - Alternative access techniques?
11	*/
12	#include <linux/atomic.h>
13	#include <linux/anon_inodes.h>
14	#include <linux/cleanup.h>
15	#include <linux/kernel.h>
16	#include <linux/export.h>
17	#include <linux/device.h>
18	#include <linux/dma-buf.h>
19	#include <linux/dma-fence.h>
20	#include <linux/dma-resv.h>
21	#include <linux/file.h>
22	#include <linux/fs.h>
23	#include <linux/cdev.h>
24	#include <linux/slab.h>
25	#include <linux/mm.h>
26	#include <linux/poll.h>
27	#include <linux/sched/signal.h>
28
29	#include <linux/iio/iio.h>
30	#include <linux/iio/iio-opaque.h>
31	#include "iio_core.h"
32	#include "iio_core_trigger.h"
33	#include <linux/iio/sysfs.h>
34	#include <linux/iio/buffer.h>
35	#include <linux/iio/buffer_impl.h>
36
37	#define DMABUF_ENQUEUE_TIMEOUT_MS 5000
38
39	MODULE_IMPORT_NS("DMA_BUF");
40
41	struct iio_dmabuf_priv {
42	struct list_head entry;
43	struct kref ref;
44
45	struct iio_buffer *buffer;
46	struct iio_dma_buffer_block *block;
47
48	u64 context;
49
50	/* Spinlock used for locking the dma_fence */
51	spinlock_t lock;
52
53	struct dma_buf_attachment *attach;
54	struct sg_table *sgt;
55	enum dma_data_direction dir;
56	atomic_t seqno;
57	};
58
59	struct iio_dma_fence {
60	struct dma_fence base;
61	struct iio_dmabuf_priv *priv;
62	struct work_struct work;
63	};
64
65	static const char * const iio_endian_prefix[] = {
66	[IIO_BE] = "be",
67	[IIO_LE] = "le",
68	};
69
70	static bool iio_buffer_is_active(struct iio_buffer *buf)
71	{
72	return !list_empty(head: &buf->buffer_list);
73	}
74
75	static size_t iio_buffer_data_available(struct iio_buffer *buf)
76	{
77	return buf->access->data_available(buf);
78	}
79
80	static int iio_buffer_flush_hwfifo(struct iio_dev *indio_dev,
81	struct iio_buffer *buf, size_t required)
82	{
83	if (!indio_dev->info->hwfifo_flush_to_buffer)
84	return -ENODEV;
85
86	return indio_dev->info->hwfifo_flush_to_buffer(indio_dev, required);
87	}
88
89	static bool iio_buffer_ready(struct iio_dev *indio_dev, struct iio_buffer *buf,
90	size_t to_wait, int to_flush)
91	{
92	size_t avail;
93	int flushed = 0;
94
95	/* wakeup if the device was unregistered */
96	if (!indio_dev->info)
97	return true;
98
99	/* drain the buffer if it was disabled */
100	if (!iio_buffer_is_active(buf)) {
101	to_wait = min_t(size_t, to_wait, 1);
102	to_flush = 0;
103	}
104
105	avail = iio_buffer_data_available(buf);
106
107	if (avail >= to_wait) {
108	/* force a flush for non-blocking reads */
109	if (!to_wait && avail < to_flush)
110	iio_buffer_flush_hwfifo(indio_dev, buf,
111	required: to_flush - avail);
112	return true;
113	}
114
115	if (to_flush)
116	flushed = iio_buffer_flush_hwfifo(indio_dev, buf,
117	required: to_wait - avail);
118	if (flushed <= 0)
119	return false;
120
121	if (avail + flushed >= to_wait)
122	return true;
123
124	return false;
125	}
126
127	/**
128	* iio_buffer_read() - chrdev read for buffer access
129	* @filp: File structure pointer for the char device
130	* @buf: Destination buffer for iio buffer read
131	* @n: First n bytes to read
132	* @f_ps: Long offset provided by the user as a seek position
133	*
134	* This function relies on all buffer implementations having an
135	* iio_buffer as their first element.
136	*
137	* Return: negative values corresponding to error codes or ret != 0
138	* for ending the reading activity
139	**/
140	static ssize_t iio_buffer_read(struct file *filp, char __user *buf,
141	size_t n, loff_t *f_ps)
142	{
143	struct iio_dev_buffer_pair *ib = filp->private_data;
144	struct iio_buffer *rb = ib->buffer;
145	struct iio_dev *indio_dev = ib->indio_dev;
146	DEFINE_WAIT_FUNC(wait, woken_wake_function);
147	size_t datum_size;
148	size_t to_wait;
149	int ret = 0;
150
151	if (!indio_dev->info)
152	return -ENODEV;
153
154	if (!rb || !rb->access->read)
155	return -EINVAL;
156
157	if (rb->direction != IIO_BUFFER_DIRECTION_IN)
158	return -EPERM;
159
160	datum_size = rb->bytes_per_datum;
161
162	/*
163	* If datum_size is 0 there will never be anything to read from the
164	* buffer, so signal end of file now.
165	*/
166	if (!datum_size)
167	return 0;
168
169	if (filp->f_flags & O_NONBLOCK)
170	to_wait = 0;
171	else
172	to_wait = min_t(size_t, n / datum_size, rb->watermark);
173
174	add_wait_queue(wq_head: &rb->pollq, wq_entry: &wait);
175	do {
176	if (!indio_dev->info) {
177	ret = -ENODEV;
178	break;
179	}
180
181	if (!iio_buffer_ready(indio_dev, buf: rb, to_wait, to_flush: n / datum_size)) {
182	if (signal_pending(current)) {
183	ret = -ERESTARTSYS;
184	break;
185	}
186
187	wait_woken(wq_entry: &wait, TASK_INTERRUPTIBLE,
188	MAX_SCHEDULE_TIMEOUT);
189	continue;
190	}
191
192	ret = rb->access->read(rb, n, buf);
193	if (ret == 0 && (filp->f_flags & O_NONBLOCK))
194	ret = -EAGAIN;
195	} while (ret == 0);
196	remove_wait_queue(wq_head: &rb->pollq, wq_entry: &wait);
197
198	return ret;
199	}
200
201	static size_t iio_buffer_space_available(struct iio_buffer *buf)
202	{
203	if (buf->access->space_available)
204	return buf->access->space_available(buf);
205
206	return SIZE_MAX;
207	}
208
209	static ssize_t iio_buffer_write(struct file *filp, const char __user *buf,
210	size_t n, loff_t *f_ps)
211	{
212	struct iio_dev_buffer_pair *ib = filp->private_data;
213	struct iio_buffer *rb = ib->buffer;
214	struct iio_dev *indio_dev = ib->indio_dev;
215	DEFINE_WAIT_FUNC(wait, woken_wake_function);
216	int ret = 0;
217	size_t written;
218
219	if (!indio_dev->info)
220	return -ENODEV;
221
222	if (!rb || !rb->access->write)
223	return -EINVAL;
224
225	if (rb->direction != IIO_BUFFER_DIRECTION_OUT)
226	return -EPERM;
227
228	written = 0;
229	add_wait_queue(wq_head: &rb->pollq, wq_entry: &wait);
230	do {
231	if (!indio_dev->info)
232	return -ENODEV;
233
234	if (!iio_buffer_space_available(buf: rb)) {
235	if (signal_pending(current)) {
236	ret = -ERESTARTSYS;
237	break;
238	}
239
240	if (filp->f_flags & O_NONBLOCK) {
241	if (!written)
242	ret = -EAGAIN;
243	break;
244	}
245
246	wait_woken(wq_entry: &wait, TASK_INTERRUPTIBLE,
247	MAX_SCHEDULE_TIMEOUT);
248	continue;
249	}
250
251	ret = rb->access->write(rb, n - written, buf + written);
252	if (ret < 0)
253	break;
254
255	written += ret;
256
257	} while (written != n);
258	remove_wait_queue(wq_head: &rb->pollq, wq_entry: &wait);
259
260	return ret < 0 ? ret : written;
261	}
262
263	/**
264	* iio_buffer_poll() - poll the buffer to find out if it has data
265	* @filp: File structure pointer for device access
266	* @wait: Poll table structure pointer for which the driver adds
267	* a wait queue
268	*
269	* Return: (EPOLLIN | EPOLLRDNORM) if data is available for reading
270	* or 0 for other cases
271	*/
272	static __poll_t iio_buffer_poll(struct file *filp,
273	struct poll_table_struct *wait)
274	{
275	struct iio_dev_buffer_pair *ib = filp->private_data;
276	struct iio_buffer *rb = ib->buffer;
277	struct iio_dev *indio_dev = ib->indio_dev;
278
279	if (!indio_dev->info || !rb)
280	return 0;
281
282	poll_wait(filp, wait_address: &rb->pollq, p: wait);
283
284	switch (rb->direction) {
285	case IIO_BUFFER_DIRECTION_IN:
286	if (iio_buffer_ready(indio_dev, buf: rb, to_wait: rb->watermark, to_flush: 0))
287	return EPOLLIN | EPOLLRDNORM;
288	break;
289	case IIO_BUFFER_DIRECTION_OUT:
290	if (iio_buffer_space_available(buf: rb))
291	return EPOLLOUT | EPOLLWRNORM;
292	break;
293	}
294
295	return 0;
296	}
297
298	ssize_t iio_buffer_read_wrapper(struct file *filp, char __user *buf,
299	size_t n, loff_t *f_ps)
300	{
301	struct iio_dev_buffer_pair *ib = filp->private_data;
302	struct iio_buffer *rb = ib->buffer;
303
304	/* check if buffer was opened through new API */
305	if (test_bit(IIO_BUSY_BIT_POS, &rb->flags))
306	return -EBUSY;
307
308	return iio_buffer_read(filp, buf, n, f_ps);
309	}
310
311	ssize_t iio_buffer_write_wrapper(struct file *filp, const char __user *buf,
312	size_t n, loff_t *f_ps)
313	{
314	struct iio_dev_buffer_pair *ib = filp->private_data;
315	struct iio_buffer *rb = ib->buffer;
316
317	/* check if buffer was opened through new API */
318	if (test_bit(IIO_BUSY_BIT_POS, &rb->flags))
319	return -EBUSY;
320
321	return iio_buffer_write(filp, buf, n, f_ps);
322	}
323
324	__poll_t iio_buffer_poll_wrapper(struct file *filp,
325	struct poll_table_struct *wait)
326	{
327	struct iio_dev_buffer_pair *ib = filp->private_data;
328	struct iio_buffer *rb = ib->buffer;
329
330	/* check if buffer was opened through new API */
331	if (test_bit(IIO_BUSY_BIT_POS, &rb->flags))
332	return 0;
333
334	return iio_buffer_poll(filp, wait);
335	}
336
337	/**
338	* iio_buffer_wakeup_poll - Wakes up the buffer waitqueue
339	* @indio_dev: The IIO device
340	*
341	* Wakes up the event waitqueue used for poll(). Should usually
342	* be called when the device is unregistered.
343	*/
344	void iio_buffer_wakeup_poll(struct iio_dev *indio_dev)
345	{
346	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
347	struct iio_buffer *buffer;
348	unsigned int i;
349
350	for (i = 0; i < iio_dev_opaque->attached_buffers_cnt; i++) {
351	buffer = iio_dev_opaque->attached_buffers[i];
352	wake_up(&buffer->pollq);
353	}
354	}
355
356	int iio_pop_from_buffer(struct iio_buffer *buffer, void *data)
357	{
358	if (!buffer || !buffer->access || !buffer->access->remove_from)
359	return -EINVAL;
360
361	return buffer->access->remove_from(buffer, data);
362	}
363	EXPORT_SYMBOL_GPL(iio_pop_from_buffer);
364
365	void iio_buffer_init(struct iio_buffer *buffer)
366	{
367	INIT_LIST_HEAD(list: &buffer->demux_list);
368	INIT_LIST_HEAD(list: &buffer->buffer_list);
369	INIT_LIST_HEAD(list: &buffer->dmabufs);
370	mutex_init(&buffer->dmabufs_mutex);
371	init_waitqueue_head(&buffer->pollq);
372	kref_init(kref: &buffer->ref);
373	if (!buffer->watermark)
374	buffer->watermark = 1;
375	}
376	EXPORT_SYMBOL(iio_buffer_init);
377
378	void iio_device_detach_buffers(struct iio_dev *indio_dev)
379	{
380	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
381	struct iio_buffer *buffer;
382	unsigned int i;
383
384	for (i = 0; i < iio_dev_opaque->attached_buffers_cnt; i++) {
385	buffer = iio_dev_opaque->attached_buffers[i];
386	iio_buffer_put(buffer);
387	}
388
389	kfree(objp: iio_dev_opaque->attached_buffers);
390	}
391
392	static ssize_t iio_show_scan_index(struct device *dev,
393	struct device_attribute *attr,
394	char *buf)
395	{
396	return sysfs_emit(buf, fmt: "%u\n", to_iio_dev_attr(attr)->c->scan_index);
397	}
398
399	static ssize_t iio_show_fixed_type(struct device *dev,
400	struct device_attribute *attr,
401	char *buf)
402	{
403	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
404	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
405	const struct iio_scan_type *scan_type;
406	u8 type;
407
408	scan_type = iio_get_current_scan_type(indio_dev, chan: this_attr->c);
409	if (IS_ERR(ptr: scan_type))
410	return PTR_ERR(ptr: scan_type);
411
412	type = scan_type->endianness;
413
414	if (type == IIO_CPU) {
415	#ifdef __LITTLE_ENDIAN
416	type = IIO_LE;
417	#else
418	type = IIO_BE;
419	#endif
420	}
421	if (scan_type->repeat > 1)
422	return sysfs_emit(buf, fmt: "%s:%c%d/%dX%d>>%u\n",
423	iio_endian_prefix[type],
424	scan_type->sign,
425	scan_type->realbits,
426	scan_type->storagebits,
427	scan_type->repeat,
428	scan_type->shift);
429	else
430	return sysfs_emit(buf, fmt: "%s:%c%d/%d>>%u\n",
431	iio_endian_prefix[type],
432	scan_type->sign,
433	scan_type->realbits,
434	scan_type->storagebits,
435	scan_type->shift);
436	}
437
438	static ssize_t iio_scan_el_show(struct device *dev,
439	struct device_attribute *attr,
440	char *buf)
441	{
442	int ret;
443	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;
444
445	/* Ensure ret is 0 or 1. */
446	ret = !!test_bit(to_iio_dev_attr(attr)->address,
447	buffer->scan_mask);
448
449	return sysfs_emit(buf, fmt: "%d\n", ret);
450	}
451
452	/* Note NULL used as error indicator as it doesn't make sense. */
453	static const unsigned long *iio_scan_mask_match(const unsigned long *av_masks,
454	unsigned int masklength,
455	const unsigned long *mask,
456	bool strict)
457	{
458	if (bitmap_empty(src: mask, nbits: masklength))
459	return NULL;
460	/*
461	* The condition here do not handle multi-long masks correctly.
462	* It only checks the first long to be zero, and will use such mask
463	* as a terminator even if there was bits set after the first long.
464	*
465	* Correct check would require using:
466	* while (!bitmap_empty(av_masks, masklength))
467	* instead. This is potentially hazardous because the
468	* avaliable_scan_masks is a zero terminated array of longs - and
469	* using the proper bitmap_empty() check for multi-long wide masks
470	* would require the array to be terminated with multiple zero longs -
471	* which is not such an usual pattern.
472	*
473	* As writing of this no multi-long wide masks were found in-tree, so
474	* the simple while (*av_masks) check is working.
475	*/
476	while (*av_masks) {
477	if (strict) {
478	if (bitmap_equal(src1: mask, src2: av_masks, nbits: masklength))
479	return av_masks;
480	} else {
481	if (bitmap_subset(src1: mask, src2: av_masks, nbits: masklength))
482	return av_masks;
483	}
484	av_masks += BITS_TO_LONGS(masklength);
485	}
486	return NULL;
487	}
488
489	static bool iio_validate_scan_mask(struct iio_dev *indio_dev,
490	const unsigned long *mask)
491	{
492	if (!indio_dev->setup_ops->validate_scan_mask)
493	return true;
494
495	return indio_dev->setup_ops->validate_scan_mask(indio_dev, mask);
496	}
497
498	/**
499	* iio_scan_mask_set() - set particular bit in the scan mask
500	* @indio_dev: the iio device
501	* @buffer: the buffer whose scan mask we are interested in
502	* @bit: the bit to be set.
503	*
504	* Note that at this point we have no way of knowing what other
505	* buffers might request, hence this code only verifies that the
506	* individual buffers request is plausible.
507	*/
508	static int iio_scan_mask_set(struct iio_dev *indio_dev,
509	struct iio_buffer *buffer, int bit)
510	{
511	unsigned int masklength = iio_get_masklength(indio_dev);
512	const unsigned long *mask;
513	unsigned long *trialmask;
514
515	if (!masklength) {
516	WARN(1, "Trying to set scanmask prior to registering buffer\n");
517	return -EINVAL;
518	}
519
520	trialmask = bitmap_alloc(nbits: masklength, GFP_KERNEL);
521	if (!trialmask)
522	return -ENOMEM;
523	bitmap_copy(dst: trialmask, src: buffer->scan_mask, nbits: masklength);
524	set_bit(nr: bit, addr: trialmask);
525
526	if (!iio_validate_scan_mask(indio_dev, mask: trialmask))
527	goto err_invalid_mask;
528
529	if (indio_dev->available_scan_masks) {
530	mask = iio_scan_mask_match(av_masks: indio_dev->available_scan_masks,
531	masklength, mask: trialmask, strict: false);
532	if (!mask)
533	goto err_invalid_mask;
534	}
535	bitmap_copy(dst: buffer->scan_mask, src: trialmask, nbits: masklength);
536
537	bitmap_free(bitmap: trialmask);
538
539	return 0;
540
541	err_invalid_mask:
542	bitmap_free(bitmap: trialmask);
543	return -EINVAL;
544	}
545
546	static int iio_scan_mask_clear(struct iio_buffer *buffer, int bit)
547	{
548	clear_bit(nr: bit, addr: buffer->scan_mask);
549	return 0;
550	}
551
552	static int iio_scan_mask_query(struct iio_dev *indio_dev,
553	struct iio_buffer *buffer, int bit)
554	{
555	if (bit > iio_get_masklength(indio_dev))
556	return -EINVAL;
557
558	if (!buffer->scan_mask)
559	return 0;
560
561	/* Ensure return value is 0 or 1. */
562	return !!test_bit(bit, buffer->scan_mask);
563	};
564
565	static ssize_t iio_scan_el_store(struct device *dev,
566	struct device_attribute *attr,
567	const char *buf,
568	size_t len)
569	{
570	int ret;
571	bool state;
572	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
573	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
574	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
575	struct iio_buffer *buffer = this_attr->buffer;
576
577	ret = kstrtobool(s: buf, res: &state);
578	if (ret < 0)
579	return ret;
580
581	guard(mutex)(T: &iio_dev_opaque->mlock);
582	if (iio_buffer_is_active(buf: buffer))
583	return -EBUSY;
584
585	ret = iio_scan_mask_query(indio_dev, buffer, bit: this_attr->address);
586	if (ret < 0)
587	return ret;
588
589	if (state && ret)
590	return len;
591
592	if (state)
593	ret = iio_scan_mask_set(indio_dev, buffer, bit: this_attr->address);
594	else
595	ret = iio_scan_mask_clear(buffer, bit: this_attr->address);
596	if (ret)
597	return ret;
598
599	return len;
600	}
601
602	static ssize_t iio_scan_el_ts_show(struct device *dev,
603	struct device_attribute *attr,
604	char *buf)
605	{
606	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;
607
608	return sysfs_emit(buf, fmt: "%d\n", buffer->scan_timestamp);
609	}
610
611	static ssize_t iio_scan_el_ts_store(struct device *dev,
612	struct device_attribute *attr,
613	const char *buf,
614	size_t len)
615	{
616	int ret;
617	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
618	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
619	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;
620	bool state;
621
622	ret = kstrtobool(s: buf, res: &state);
623	if (ret < 0)
624	return ret;
625
626	guard(mutex)(T: &iio_dev_opaque->mlock);
627	if (iio_buffer_is_active(buf: buffer))
628	return -EBUSY;
629
630	buffer->scan_timestamp = state;
631
632	return len;
633	}
634
635	static int iio_buffer_add_channel_sysfs(struct iio_dev *indio_dev,
636	struct iio_buffer *buffer,
637	const struct iio_chan_spec *chan)
638	{
639	int ret, attrcount = 0;
640
641	ret = __iio_add_chan_devattr(postfix: "index",
642	chan,
643	func: &iio_show_scan_index,
644	NULL,
645	mask: 0,
646	shared_by: IIO_SEPARATE,
647	dev: &indio_dev->dev,
648	buffer,
649	attr_list: &buffer->buffer_attr_list);
650	if (ret)
651	return ret;
652	attrcount++;
653	ret = __iio_add_chan_devattr(postfix: "type",
654	chan,
655	func: &iio_show_fixed_type,
656	NULL,
657	mask: 0,
658	shared_by: IIO_SEPARATE,
659	dev: &indio_dev->dev,
660	buffer,
661	attr_list: &buffer->buffer_attr_list);
662	if (ret)
663	return ret;
664	attrcount++;
665	if (chan->type != IIO_TIMESTAMP)
666	ret = __iio_add_chan_devattr(postfix: "en",
667	chan,
668	func: &iio_scan_el_show,
669	writefunc: &iio_scan_el_store,
670	mask: chan->scan_index,
671	shared_by: IIO_SEPARATE,
672	dev: &indio_dev->dev,
673	buffer,
674	attr_list: &buffer->buffer_attr_list);
675	else
676	ret = __iio_add_chan_devattr(postfix: "en",
677	chan,
678	func: &iio_scan_el_ts_show,
679	writefunc: &iio_scan_el_ts_store,
680	mask: chan->scan_index,
681	shared_by: IIO_SEPARATE,
682	dev: &indio_dev->dev,
683	buffer,
684	attr_list: &buffer->buffer_attr_list);
685	if (ret)
686	return ret;
687	attrcount++;
688	ret = attrcount;
689	return ret;
690	}
691
692	static ssize_t length_show(struct device *dev, struct device_attribute *attr,
693	char *buf)
694	{
695	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;
696
697	return sysfs_emit(buf, fmt: "%d\n", buffer->length);
698	}
699
700	static ssize_t length_store(struct device *dev, struct device_attribute *attr,
701	const char *buf, size_t len)
702	{
703	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
704	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
705	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;
706	unsigned int val;
707	int ret;
708
709	ret = kstrtouint(s: buf, base: 10, res: &val);
710	if (ret)
711	return ret;
712
713	if (val == buffer->length)
714	return len;
715
716	guard(mutex)(T: &iio_dev_opaque->mlock);
717	if (iio_buffer_is_active(buf: buffer))
718	return -EBUSY;
719
720	buffer->access->set_length(buffer, val);
721
722	if (buffer->length && buffer->length < buffer->watermark)
723	buffer->watermark = buffer->length;
724
725	return len;
726	}
727
728	static ssize_t enable_show(struct device *dev, struct device_attribute *attr,
729	char *buf)
730	{
731	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;
732
733	return sysfs_emit(buf, fmt: "%d\n", iio_buffer_is_active(buf: buffer));
734	}
735
736	static int iio_storage_bytes_for_si(struct iio_dev *indio_dev,
737	unsigned int scan_index)
738	{
739	const struct iio_chan_spec *ch;
740	const struct iio_scan_type *scan_type;
741	unsigned int bytes;
742
743	ch = iio_find_channel_from_si(indio_dev, si: scan_index);
744	scan_type = iio_get_current_scan_type(indio_dev, chan: ch);
745	if (IS_ERR(ptr: scan_type))
746	return PTR_ERR(ptr: scan_type);
747
748	bytes = scan_type->storagebits / 8;
749
750	if (scan_type->repeat > 1)
751	bytes *= scan_type->repeat;
752
753	return bytes;
754	}
755
756	static int iio_storage_bytes_for_timestamp(struct iio_dev *indio_dev)
757	{
758	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
759
760	return iio_storage_bytes_for_si(indio_dev,
761	scan_index: iio_dev_opaque->scan_index_timestamp);
762	}
763
764	static int iio_compute_scan_bytes(struct iio_dev *indio_dev,
765	const unsigned long *mask, bool timestamp)
766	{
767	unsigned int bytes = 0;
768	int length, i, largest = 0;
769
770	/* How much space will the demuxed element take? */
771	for_each_set_bit(i, mask, iio_get_masklength(indio_dev)) {
772	length = iio_storage_bytes_for_si(indio_dev, scan_index: i);
773	if (length < 0)
774	return length;
775
776	bytes = ALIGN(bytes, length);
777	bytes += length;
778	largest = max(largest, length);
779	}
780
781	if (timestamp) {
782	length = iio_storage_bytes_for_timestamp(indio_dev);
783	if (length < 0)
784	return length;
785
786	bytes = ALIGN(bytes, length);
787	bytes += length;
788	largest = max(largest, length);
789	}
790
791	bytes = ALIGN(bytes, largest);
792	return bytes;
793	}
794
795	static void iio_buffer_activate(struct iio_dev *indio_dev,
796	struct iio_buffer *buffer)
797	{
798	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
799
800	iio_buffer_get(buffer);
801	list_add(new: &buffer->buffer_list, head: &iio_dev_opaque->buffer_list);
802	}
803
804	static void iio_buffer_deactivate(struct iio_buffer *buffer)
805	{
806	list_del_init(entry: &buffer->buffer_list);
807	wake_up_interruptible(&buffer->pollq);
808	iio_buffer_put(buffer);
809	}
810
811	static void iio_buffer_deactivate_all(struct iio_dev *indio_dev)
812	{
813	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
814	struct iio_buffer *buffer, *_buffer;
815
816	list_for_each_entry_safe(buffer, _buffer,
817	&iio_dev_opaque->buffer_list, buffer_list)
818	iio_buffer_deactivate(buffer);
819	}
820
821	static int iio_buffer_enable(struct iio_buffer *buffer,
822	struct iio_dev *indio_dev)
823	{
824	if (!buffer->access->enable)
825	return 0;
826	return buffer->access->enable(buffer, indio_dev);
827	}
828
829	static int iio_buffer_disable(struct iio_buffer *buffer,
830	struct iio_dev *indio_dev)
831	{
832	if (!buffer->access->disable)
833	return 0;
834	return buffer->access->disable(buffer, indio_dev);
835	}
836
837	static void iio_buffer_update_bytes_per_datum(struct iio_dev *indio_dev,
838	struct iio_buffer *buffer)
839	{
840	unsigned int bytes;
841
842	if (!buffer->access->set_bytes_per_datum)
843	return;
844
845	bytes = iio_compute_scan_bytes(indio_dev, mask: buffer->scan_mask,
846	timestamp: buffer->scan_timestamp);
847
848	buffer->access->set_bytes_per_datum(buffer, bytes);
849	}
850
851	static int iio_buffer_request_update(struct iio_dev *indio_dev,
852	struct iio_buffer *buffer)
853	{
854	int ret;
855
856	iio_buffer_update_bytes_per_datum(indio_dev, buffer);
857	if (buffer->access->request_update) {
858	ret = buffer->access->request_update(buffer);
859	if (ret) {
860	dev_dbg(&indio_dev->dev,
861	"Buffer not started: buffer parameter update failed (%d)\n",
862	ret);
863	return ret;
864	}
865	}
866
867	return 0;
868	}
869
870	static void iio_free_scan_mask(struct iio_dev *indio_dev,
871	const unsigned long *mask)
872	{
873	/* If the mask is dynamically allocated free it, otherwise do nothing */
874	if (!indio_dev->available_scan_masks)
875	bitmap_free(bitmap: mask);
876	}
877
878	struct iio_device_config {
879	unsigned int mode;
880	unsigned int watermark;
881	const unsigned long *scan_mask;
882	unsigned int scan_bytes;
883	bool scan_timestamp;
884	};
885
886	static int iio_verify_update(struct iio_dev *indio_dev,
887	struct iio_buffer *insert_buffer,
888	struct iio_buffer *remove_buffer,
889	struct iio_device_config *config)
890	{
891	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
892	unsigned int masklength = iio_get_masklength(indio_dev);
893	unsigned long *compound_mask;
894	const unsigned long *scan_mask;
895	bool strict_scanmask = false;
896	struct iio_buffer *buffer;
897	bool scan_timestamp;
898	unsigned int modes;
899
900	if (insert_buffer &&
901	bitmap_empty(src: insert_buffer->scan_mask, nbits: masklength)) {
902	dev_dbg(&indio_dev->dev,
903	"At least one scan element must be enabled first\n");
904	return -EINVAL;
905	}
906
907	memset(config, 0, sizeof(*config));
908	config->watermark = ~0;
909
910	/*
911	* If there is just one buffer and we are removing it there is nothing
912	* to verify.
913	*/
914	if (remove_buffer && !insert_buffer &&
915	list_is_singular(head: &iio_dev_opaque->buffer_list))
916	return 0;
917
918	modes = indio_dev->modes;
919
920	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {
921	if (buffer == remove_buffer)
922	continue;
923	modes &= buffer->access->modes;
924	config->watermark = min(config->watermark, buffer->watermark);
925	}
926
927	if (insert_buffer) {
928	modes &= insert_buffer->access->modes;
929	config->watermark = min(config->watermark,
930	insert_buffer->watermark);
931	}
932
933	/* Definitely possible for devices to support both of these. */
934	if ((modes & INDIO_BUFFER_TRIGGERED) && indio_dev->trig) {
935	config->mode = INDIO_BUFFER_TRIGGERED;
936	} else if (modes & INDIO_BUFFER_HARDWARE) {
937	/*
938	* Keep things simple for now and only allow a single buffer to
939	* be connected in hardware mode.
940	*/
941	if (insert_buffer && !list_empty(head: &iio_dev_opaque->buffer_list))
942	return -EINVAL;
943	config->mode = INDIO_BUFFER_HARDWARE;
944	strict_scanmask = true;
945	} else if (modes & INDIO_BUFFER_SOFTWARE) {
946	config->mode = INDIO_BUFFER_SOFTWARE;
947	} else {
948	/* Can only occur on first buffer */
949	if (indio_dev->modes & INDIO_BUFFER_TRIGGERED)
950	dev_dbg(&indio_dev->dev, "Buffer not started: no trigger\n");
951	return -EINVAL;
952	}
953
954	/* What scan mask do we actually have? */
955	compound_mask = bitmap_zalloc(nbits: masklength, GFP_KERNEL);
956	if (!compound_mask)
957	return -ENOMEM;
958
959	scan_timestamp = false;
960
961	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {
962	if (buffer == remove_buffer)
963	continue;
964	bitmap_or(dst: compound_mask, src1: compound_mask, src2: buffer->scan_mask,
965	nbits: masklength);
966	scan_timestamp |= buffer->scan_timestamp;
967	}
968
969	if (insert_buffer) {
970	bitmap_or(dst: compound_mask, src1: compound_mask,
971	src2: insert_buffer->scan_mask, nbits: masklength);
972	scan_timestamp |= insert_buffer->scan_timestamp;
973	}
974
975	if (indio_dev->available_scan_masks) {
976	scan_mask = iio_scan_mask_match(av_masks: indio_dev->available_scan_masks,
977	masklength, mask: compound_mask,
978	strict: strict_scanmask);
979	bitmap_free(bitmap: compound_mask);
980	if (!scan_mask)
981	return -EINVAL;
982	} else {
983	scan_mask = compound_mask;
984	}
985
986	config->scan_bytes = iio_compute_scan_bytes(indio_dev,
987	mask: scan_mask, timestamp: scan_timestamp);
988	config->scan_mask = scan_mask;
989	config->scan_timestamp = scan_timestamp;
990
991	return 0;
992	}
993
994	/**
995	* struct iio_demux_table - table describing demux memcpy ops
996	* @from: index to copy from
997	* @to: index to copy to
998	* @length: how many bytes to copy
999	* @l: list head used for management
1000	*/
1001	struct iio_demux_table {
1002	unsigned int from;
1003	unsigned int to;
1004	unsigned int length;
1005	struct list_head l;
1006	};
1007
1008	static void iio_buffer_demux_free(struct iio_buffer *buffer)
1009	{
1010	struct iio_demux_table *p, *q;
1011
1012	list_for_each_entry_safe(p, q, &buffer->demux_list, l) {
1013	list_del(entry: &p->l);
1014	kfree(objp: p);
1015	}
1016	}
1017
1018	static int iio_buffer_add_demux(struct iio_buffer *buffer,
1019	struct iio_demux_table **p, unsigned int in_loc,
1020	unsigned int out_loc,
1021	unsigned int length)
1022	{
1023	if (*p && (*p)->from + (*p)->length == in_loc &&
1024	(*p)->to + (*p)->length == out_loc) {
1025	(*p)->length += length;
1026	} else {
1027	*p = kmalloc(sizeof(**p), GFP_KERNEL);
1028	if (!(*p))
1029	return -ENOMEM;
1030	(*p)->from = in_loc;
1031	(*p)->to = out_loc;
1032	(*p)->length = length;
1033	list_add_tail(new: &(*p)->l, head: &buffer->demux_list);
1034	}
1035
1036	return 0;
1037	}
1038
1039	static int iio_buffer_update_demux(struct iio_dev *indio_dev,
1040	struct iio_buffer *buffer)
1041	{
1042	unsigned int masklength = iio_get_masklength(indio_dev);
1043	int ret, in_ind = -1, out_ind, length;
1044	unsigned int in_loc = 0, out_loc = 0;
1045	struct iio_demux_table *p = NULL;
1046
1047	/* Clear out any old demux */
1048	iio_buffer_demux_free(buffer);
1049	kfree(objp: buffer->demux_bounce);
1050	buffer->demux_bounce = NULL;
1051
1052	/* First work out which scan mode we will actually have */
1053	if (bitmap_equal(src1: indio_dev->active_scan_mask,
1054	src2: buffer->scan_mask, nbits: masklength))
1055	return 0;
1056
1057	/* Now we have the two masks, work from least sig and build up sizes */
1058	for_each_set_bit(out_ind, buffer->scan_mask, masklength) {
1059	in_ind = find_next_bit(addr: indio_dev->active_scan_mask,
1060	size: masklength, offset: in_ind + 1);
1061	while (in_ind != out_ind) {
1062	ret = iio_storage_bytes_for_si(indio_dev, scan_index: in_ind);
1063	if (ret < 0)
1064	goto error_clear_mux_table;
1065
1066	length = ret;
1067	/* Make sure we are aligned */
1068	in_loc = roundup(in_loc, length) + length;
1069	in_ind = find_next_bit(addr: indio_dev->active_scan_mask,
1070	size: masklength, offset: in_ind + 1);
1071	}
1072	ret = iio_storage_bytes_for_si(indio_dev, scan_index: in_ind);
1073	if (ret < 0)
1074	goto error_clear_mux_table;
1075
1076	length = ret;
1077	out_loc = roundup(out_loc, length);
1078	in_loc = roundup(in_loc, length);
1079	ret = iio_buffer_add_demux(buffer, p: &p, in_loc, out_loc, length);
1080	if (ret)
1081	goto error_clear_mux_table;
1082	out_loc += length;
1083	in_loc += length;
1084	}
1085	/* Relies on scan_timestamp being last */
1086	if (buffer->scan_timestamp) {
1087	ret = iio_storage_bytes_for_timestamp(indio_dev);
1088	if (ret < 0)
1089	goto error_clear_mux_table;
1090
1091	length = ret;
1092	out_loc = roundup(out_loc, length);
1093	in_loc = roundup(in_loc, length);
1094	ret = iio_buffer_add_demux(buffer, p: &p, in_loc, out_loc, length);
1095	if (ret)
1096	goto error_clear_mux_table;
1097	out_loc += length;
1098	}
1099	buffer->demux_bounce = kzalloc(out_loc, GFP_KERNEL);
1100	if (!buffer->demux_bounce) {
1101	ret = -ENOMEM;
1102	goto error_clear_mux_table;
1103	}
1104	return 0;
1105
1106	error_clear_mux_table:
1107	iio_buffer_demux_free(buffer);
1108
1109	return ret;
1110	}
1111
1112	static int iio_update_demux(struct iio_dev *indio_dev)
1113	{
1114	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
1115	struct iio_buffer *buffer;
1116	int ret;
1117
1118	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {
1119	ret = iio_buffer_update_demux(indio_dev, buffer);
1120	if (ret < 0)
1121	goto error_clear_mux_table;
1122	}
1123	return 0;
1124
1125	error_clear_mux_table:
1126	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list)
1127	iio_buffer_demux_free(buffer);
1128
1129	return ret;
1130	}
1131
1132	static int iio_enable_buffers(struct iio_dev *indio_dev,
1133	struct iio_device_config *config)
1134	{
1135	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
1136	struct iio_buffer *buffer, *tmp = NULL;
1137	int ret;
1138
1139	indio_dev->active_scan_mask = config->scan_mask;
1140	ACCESS_PRIVATE(indio_dev, scan_timestamp) = config->scan_timestamp;
1141	indio_dev->scan_bytes = config->scan_bytes;
1142	iio_dev_opaque->currentmode = config->mode;
1143
1144	iio_update_demux(indio_dev);
1145
1146	/* Wind up again */
1147	if (indio_dev->setup_ops->preenable) {
1148	ret = indio_dev->setup_ops->preenable(indio_dev);
1149	if (ret) {
1150	dev_dbg(&indio_dev->dev,
1151	"Buffer not started: buffer preenable failed (%d)\n", ret);
1152	goto err_undo_config;
1153	}
1154	}
1155
1156	if (indio_dev->info->update_scan_mode) {
1157	ret = indio_dev->info
1158	->update_scan_mode(indio_dev,
1159	indio_dev->active_scan_mask);
1160	if (ret < 0) {
1161	dev_dbg(&indio_dev->dev,
1162	"Buffer not started: update scan mode failed (%d)\n",
1163	ret);
1164	goto err_run_postdisable;
1165	}
1166	}
1167
1168	if (indio_dev->info->hwfifo_set_watermark)
1169	indio_dev->info->hwfifo_set_watermark(indio_dev,
1170	config->watermark);
1171
1172	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {
1173	ret = iio_buffer_enable(buffer, indio_dev);
1174	if (ret) {
1175	tmp = buffer;
1176	goto err_disable_buffers;
1177	}
1178	}
1179
1180	if (iio_dev_opaque->currentmode == INDIO_BUFFER_TRIGGERED) {
1181	ret = iio_trigger_attach_poll_func(trig: indio_dev->trig,
1182	pf: indio_dev->pollfunc);
1183	if (ret)
1184	goto err_disable_buffers;
1185	}
1186
1187	if (indio_dev->setup_ops->postenable) {
1188	ret = indio_dev->setup_ops->postenable(indio_dev);
1189	if (ret) {
1190	dev_dbg(&indio_dev->dev,
1191	"Buffer not started: postenable failed (%d)\n", ret);
1192	goto err_detach_pollfunc;
1193	}
1194	}
1195
1196	return 0;
1197
1198	err_detach_pollfunc:
1199	if (iio_dev_opaque->currentmode == INDIO_BUFFER_TRIGGERED) {
1200	iio_trigger_detach_poll_func(trig: indio_dev->trig,
1201	pf: indio_dev->pollfunc);
1202	}
1203	err_disable_buffers:
1204	buffer = list_prepare_entry(tmp, &iio_dev_opaque->buffer_list, buffer_list);
1205	list_for_each_entry_continue_reverse(buffer, &iio_dev_opaque->buffer_list,
1206	buffer_list)
1207	iio_buffer_disable(buffer, indio_dev);
1208	err_run_postdisable:
1209	if (indio_dev->setup_ops->postdisable)
1210	indio_dev->setup_ops->postdisable(indio_dev);
1211	err_undo_config:
1212	iio_dev_opaque->currentmode = INDIO_DIRECT_MODE;
1213	indio_dev->active_scan_mask = NULL;
1214
1215	return ret;
1216	}
1217
1218	static int iio_disable_buffers(struct iio_dev *indio_dev)
1219	{
1220	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
1221	struct iio_buffer *buffer;
1222	int ret = 0;
1223	int ret2;
1224
1225	/* Wind down existing buffers - iff there are any */
1226	if (list_empty(head: &iio_dev_opaque->buffer_list))
1227	return 0;
1228
1229	/*
1230	* If things go wrong at some step in disable we still need to continue
1231	* to perform the other steps, otherwise we leave the device in a
1232	* inconsistent state. We return the error code for the first error we
1233	* encountered.
1234	*/
1235
1236	if (indio_dev->setup_ops->predisable) {
1237	ret2 = indio_dev->setup_ops->predisable(indio_dev);
1238	if (ret2 && !ret)
1239	ret = ret2;
1240	}
1241
1242	if (iio_dev_opaque->currentmode == INDIO_BUFFER_TRIGGERED) {
1243	iio_trigger_detach_poll_func(trig: indio_dev->trig,
1244	pf: indio_dev->pollfunc);
1245	}
1246
1247	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {
1248	ret2 = iio_buffer_disable(buffer, indio_dev);
1249	if (ret2 && !ret)
1250	ret = ret2;
1251	}
1252
1253	if (indio_dev->setup_ops->postdisable) {
1254	ret2 = indio_dev->setup_ops->postdisable(indio_dev);
1255	if (ret2 && !ret)
1256	ret = ret2;
1257	}
1258
1259	iio_free_scan_mask(indio_dev, mask: indio_dev->active_scan_mask);
1260	indio_dev->active_scan_mask = NULL;
1261	iio_dev_opaque->currentmode = INDIO_DIRECT_MODE;
1262
1263	return ret;
1264	}
1265
1266	static int __iio_update_buffers(struct iio_dev *indio_dev,
1267	struct iio_buffer *insert_buffer,
1268	struct iio_buffer *remove_buffer)
1269	{
1270	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
1271	struct iio_device_config new_config;
1272	int ret;
1273
1274	ret = iio_verify_update(indio_dev, insert_buffer, remove_buffer,
1275	config: &new_config);
1276	if (ret)
1277	return ret;
1278
1279	if (insert_buffer) {
1280	ret = iio_buffer_request_update(indio_dev, buffer: insert_buffer);
1281	if (ret)
1282	goto err_free_config;
1283	}
1284
1285	ret = iio_disable_buffers(indio_dev);
1286	if (ret)
1287	goto err_deactivate_all;
1288
1289	if (remove_buffer)
1290	iio_buffer_deactivate(buffer: remove_buffer);
1291	if (insert_buffer)
1292	iio_buffer_activate(indio_dev, buffer: insert_buffer);
1293
1294	/* If no buffers in list, we are done */
1295	if (list_empty(head: &iio_dev_opaque->buffer_list))
1296	return 0;
1297
1298	ret = iio_enable_buffers(indio_dev, config: &new_config);
1299	if (ret)
1300	goto err_deactivate_all;
1301
1302	return 0;
1303
1304	err_deactivate_all:
1305	/*
1306	* We've already verified that the config is valid earlier. If things go
1307	* wrong in either enable or disable the most likely reason is an IO
1308	* error from the device. In this case there is no good recovery
1309	* strategy. Just make sure to disable everything and leave the device
1310	* in a sane state. With a bit of luck the device might come back to
1311	* life again later and userspace can try again.
1312	*/
1313	iio_buffer_deactivate_all(indio_dev);
1314
1315	err_free_config:
1316	iio_free_scan_mask(indio_dev, mask: new_config.scan_mask);
1317	return ret;
1318	}
1319
1320	int iio_update_buffers(struct iio_dev *indio_dev,
1321	struct iio_buffer *insert_buffer,
1322	struct iio_buffer *remove_buffer)
1323	{
1324	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
1325
1326	if (insert_buffer == remove_buffer)
1327	return 0;
1328
1329	if (insert_buffer &&
1330	insert_buffer->direction == IIO_BUFFER_DIRECTION_OUT)
1331	return -EINVAL;
1332
1333	guard(mutex)(T: &iio_dev_opaque->info_exist_lock);
1334	guard(mutex)(T: &iio_dev_opaque->mlock);
1335
1336	if (insert_buffer && iio_buffer_is_active(buf: insert_buffer))
1337	insert_buffer = NULL;
1338
1339	if (remove_buffer && !iio_buffer_is_active(buf: remove_buffer))
1340	remove_buffer = NULL;
1341
1342	if (!insert_buffer && !remove_buffer)
1343	return 0;
1344
1345	if (!indio_dev->info)
1346	return -ENODEV;
1347
1348	return __iio_update_buffers(indio_dev, insert_buffer, remove_buffer);
1349	}
1350	EXPORT_SYMBOL_GPL(iio_update_buffers);
1351
1352	void iio_disable_all_buffers(struct iio_dev *indio_dev)
1353	{
1354	iio_disable_buffers(indio_dev);
1355	iio_buffer_deactivate_all(indio_dev);
1356	}
1357
1358	static ssize_t enable_store(struct device *dev, struct device_attribute *attr,
1359	const char *buf, size_t len)
1360	{
1361	int ret;
1362	bool requested_state;
1363	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1364	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
1365	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;
1366	bool inlist;
1367
1368	ret = kstrtobool(s: buf, res: &requested_state);
1369	if (ret < 0)
1370	return ret;
1371
1372	guard(mutex)(T: &iio_dev_opaque->mlock);
1373
1374	/* Find out if it is in the list */
1375	inlist = iio_buffer_is_active(buf: buffer);
1376	/* Already in desired state */
1377	if (inlist == requested_state)
1378	return len;
1379
1380	if (requested_state)
1381	ret = __iio_update_buffers(indio_dev, insert_buffer: buffer, NULL);
1382	else
1383	ret = __iio_update_buffers(indio_dev, NULL, remove_buffer: buffer);
1384	if (ret)
1385	return ret;
1386
1387	return len;
1388	}
1389
1390	static ssize_t watermark_show(struct device *dev, struct device_attribute *attr,
1391	char *buf)
1392	{
1393	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;
1394
1395	return sysfs_emit(buf, fmt: "%u\n", buffer->watermark);
1396	}
1397
1398	static ssize_t watermark_store(struct device *dev,
1399	struct device_attribute *attr,
1400	const char *buf, size_t len)
1401	{
1402	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1403	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
1404	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;
1405	unsigned int val;
1406	int ret;
1407
1408	ret = kstrtouint(s: buf, base: 10, res: &val);
1409	if (ret)
1410	return ret;
1411	if (!val)
1412	return -EINVAL;
1413
1414	guard(mutex)(T: &iio_dev_opaque->mlock);
1415
1416	if (val > buffer->length)
1417	return -EINVAL;
1418
1419	if (iio_buffer_is_active(buf: buffer))
1420	return -EBUSY;
1421
1422	buffer->watermark = val;
1423
1424	return len;
1425	}
1426
1427	static ssize_t data_available_show(struct device *dev,
1428	struct device_attribute *attr, char *buf)
1429	{
1430	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;
1431
1432	return sysfs_emit(buf, fmt: "%zu\n", iio_buffer_data_available(buf: buffer));
1433	}
1434
1435	static ssize_t direction_show(struct device *dev,
1436	struct device_attribute *attr,
1437	char *buf)
1438	{
1439	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;
1440
1441	switch (buffer->direction) {
1442	case IIO_BUFFER_DIRECTION_IN:
1443	return sysfs_emit(buf, fmt: "in\n");
1444	case IIO_BUFFER_DIRECTION_OUT:
1445	return sysfs_emit(buf, fmt: "out\n");
1446	default:
1447	return -EINVAL;
1448	}
1449	}
1450
1451	static DEVICE_ATTR_RW(length);
1452	static struct device_attribute dev_attr_length_ro = __ATTR_RO(length);
1453	static DEVICE_ATTR_RW(enable);
1454	static DEVICE_ATTR_RW(watermark);
1455	static struct device_attribute dev_attr_watermark_ro = __ATTR_RO(watermark);
1456	static DEVICE_ATTR_RO(data_available);
1457	static DEVICE_ATTR_RO(direction);
1458
1459	/*
1460	* When adding new attributes here, put the at the end, at least until
1461	* the code that handles the length/length_ro & watermark/watermark_ro
1462	* assignments gets cleaned up. Otherwise these can create some weird
1463	* duplicate attributes errors under some setups.
1464	*/
1465	static struct attribute *iio_buffer_attrs[] = {
1466	&dev_attr_length.attr,
1467	&dev_attr_enable.attr,
1468	&dev_attr_watermark.attr,
1469	&dev_attr_data_available.attr,
1470	&dev_attr_direction.attr,
1471	};
1472
1473	#define to_dev_attr(_attr) container_of(_attr, struct device_attribute, attr)
1474
1475	static struct attribute *iio_buffer_wrap_attr(struct iio_buffer *buffer,
1476	struct attribute *attr)
1477	{
1478	struct device_attribute *dattr = to_dev_attr(attr);
1479	struct iio_dev_attr *iio_attr;
1480
1481	iio_attr = kzalloc(sizeof(*iio_attr), GFP_KERNEL);
1482	if (!iio_attr)
1483	return NULL;
1484
1485	iio_attr->buffer = buffer;
1486	memcpy(&iio_attr->dev_attr, dattr, sizeof(iio_attr->dev_attr));
1487	iio_attr->dev_attr.attr.name = kstrdup_const(s: attr->name, GFP_KERNEL);
1488	if (!iio_attr->dev_attr.attr.name) {
1489	kfree(objp: iio_attr);
1490	return NULL;
1491	}
1492
1493	sysfs_attr_init(&iio_attr->dev_attr.attr);
1494
1495	list_add(new: &iio_attr->l, head: &buffer->buffer_attr_list);
1496
1497	return &iio_attr->dev_attr.attr;
1498	}
1499
1500	static int iio_buffer_register_legacy_sysfs_groups(struct iio_dev *indio_dev,
1501	struct attribute **buffer_attrs,
1502	int buffer_attrcount,
1503	int scan_el_attrcount)
1504	{
1505	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
1506	struct attribute_group *group;
1507	struct attribute **attrs;
1508	int ret;
1509
1510	attrs = kcalloc(buffer_attrcount + 1, sizeof(*attrs), GFP_KERNEL);
1511	if (!attrs)
1512	return -ENOMEM;
1513
1514	memcpy(attrs, buffer_attrs, buffer_attrcount * sizeof(*attrs));
1515
1516	group = &iio_dev_opaque->legacy_buffer_group;
1517	group->attrs = attrs;
1518	group->name = "buffer";
1519
1520	ret = iio_device_register_sysfs_group(indio_dev, group);
1521	if (ret)
1522	goto error_free_buffer_attrs;
1523
1524	attrs = kcalloc(scan_el_attrcount + 1, sizeof(*attrs), GFP_KERNEL);
1525	if (!attrs) {
1526	ret = -ENOMEM;
1527	goto error_free_buffer_attrs;
1528	}
1529
1530	memcpy(attrs, &buffer_attrs[buffer_attrcount],
1531	scan_el_attrcount * sizeof(*attrs));
1532
1533	group = &iio_dev_opaque->legacy_scan_el_group;
1534	group->attrs = attrs;
1535	group->name = "scan_elements";
1536
1537	ret = iio_device_register_sysfs_group(indio_dev, group);
1538	if (ret)
1539	goto error_free_scan_el_attrs;
1540
1541	return 0;
1542
1543	error_free_scan_el_attrs:
1544	kfree(objp: iio_dev_opaque->legacy_scan_el_group.attrs);
1545	error_free_buffer_attrs:
1546	kfree(objp: iio_dev_opaque->legacy_buffer_group.attrs);
1547
1548	return ret;
1549	}
1550
1551	static void iio_buffer_unregister_legacy_sysfs_groups(struct iio_dev *indio_dev)
1552	{
1553	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
1554
1555	kfree(objp: iio_dev_opaque->legacy_buffer_group.attrs);
1556	kfree(objp: iio_dev_opaque->legacy_scan_el_group.attrs);
1557	}
1558
1559	static void iio_buffer_dmabuf_release(struct kref *ref)
1560	{
1561	struct iio_dmabuf_priv *priv = container_of(ref, struct iio_dmabuf_priv, ref);
1562	struct dma_buf_attachment *attach = priv->attach;
1563	struct iio_buffer *buffer = priv->buffer;
1564	struct dma_buf *dmabuf = attach->dmabuf;
1565
1566	dma_buf_unmap_attachment_unlocked(attach, sg_table: priv->sgt, direction: priv->dir);
1567
1568	buffer->access->detach_dmabuf(buffer, priv->block);
1569
1570	dma_buf_detach(dmabuf: attach->dmabuf, attach);
1571	dma_buf_put(dmabuf);
1572	kfree(objp: priv);
1573	}
1574
1575	static void iio_buffer_dmabuf_get(struct dma_buf_attachment *attach)
1576	{
1577	struct iio_dmabuf_priv *priv = attach->importer_priv;
1578
1579	kref_get(kref: &priv->ref);
1580	}
1581
1582	static void iio_buffer_dmabuf_put(struct dma_buf_attachment *attach)
1583	{
1584	struct iio_dmabuf_priv *priv = attach->importer_priv;
1585
1586	kref_put(kref: &priv->ref, release: iio_buffer_dmabuf_release);
1587	}
1588
1589	static int iio_buffer_chrdev_release(struct inode *inode, struct file *filep)
1590	{
1591	struct iio_dev_buffer_pair *ib = filep->private_data;
1592	struct iio_dev *indio_dev = ib->indio_dev;
1593	struct iio_buffer *buffer = ib->buffer;
1594	struct iio_dmabuf_priv *priv, *tmp;
1595
1596	wake_up(&buffer->pollq);
1597
1598	guard(mutex)(T: &buffer->dmabufs_mutex);
1599
1600	/* Close all attached DMABUFs */
1601	list_for_each_entry_safe(priv, tmp, &buffer->dmabufs, entry) {
1602	list_del_init(entry: &priv->entry);
1603	iio_buffer_dmabuf_put(attach: priv->attach);
1604	}
1605
1606	kfree(objp: ib);
1607	clear_bit(IIO_BUSY_BIT_POS, addr: &buffer->flags);
1608	iio_device_put(indio_dev);
1609
1610	return 0;
1611	}
1612
1613	static int iio_dma_resv_lock(struct dma_buf *dmabuf, bool nonblock)
1614	{
1615	if (!nonblock)
1616	return dma_resv_lock_interruptible(obj: dmabuf->resv, NULL);
1617
1618	if (!dma_resv_trylock(obj: dmabuf->resv))
1619	return -EBUSY;
1620
1621	return 0;
1622	}
1623
1624	static struct device *iio_buffer_get_dma_dev(const struct iio_dev *indio_dev,
1625	struct iio_buffer *buffer)
1626	{
1627	if (buffer->access->get_dma_dev)
1628	return buffer->access->get_dma_dev(buffer);
1629
1630	return indio_dev->dev.parent;
1631	}
1632
1633	static struct dma_buf_attachment *
1634	iio_buffer_find_attachment(struct iio_dev_buffer_pair *ib,
1635	struct dma_buf *dmabuf, bool nonblock)
1636	{
1637	struct iio_buffer *buffer = ib->buffer;
1638	struct device *dma_dev = iio_buffer_get_dma_dev(indio_dev: ib->indio_dev, buffer);
1639	struct dma_buf_attachment *attach = NULL;
1640	struct iio_dmabuf_priv *priv;
1641
1642	guard(mutex)(T: &buffer->dmabufs_mutex);
1643
1644	list_for_each_entry(priv, &buffer->dmabufs, entry) {
1645	if (priv->attach->dev == dma_dev
1646	&& priv->attach->dmabuf == dmabuf) {
1647	attach = priv->attach;
1648	break;
1649	}
1650	}
1651
1652	if (attach)
1653	iio_buffer_dmabuf_get(attach);
1654
1655	return attach ?: ERR_PTR(error: -EPERM);
1656	}
1657
1658	static int iio_buffer_attach_dmabuf(struct iio_dev_buffer_pair *ib,
1659	int __user *user_fd, bool nonblock)
1660	{
1661	struct iio_dev *indio_dev = ib->indio_dev;
1662	struct iio_buffer *buffer = ib->buffer;
1663	struct device *dma_dev = iio_buffer_get_dma_dev(indio_dev, buffer);
1664	struct dma_buf_attachment *attach;
1665	struct iio_dmabuf_priv *priv, *each;
1666	struct dma_buf *dmabuf;
1667	int err, fd;
1668
1669	if (!buffer->access->attach_dmabuf
1670	|| !buffer->access->detach_dmabuf
1671	|| !buffer->access->enqueue_dmabuf)
1672	return -EPERM;
1673
1674	if (copy_from_user(to: &fd, from: user_fd, n: sizeof(fd)))
1675	return -EFAULT;
1676
1677	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1678	if (!priv)
1679	return -ENOMEM;
1680
1681	spin_lock_init(&priv->lock);
1682	priv->context = dma_fence_context_alloc(num: 1);
1683
1684	dmabuf = dma_buf_get(fd);
1685	if (IS_ERR(ptr: dmabuf)) {
1686	err = PTR_ERR(ptr: dmabuf);
1687	goto err_free_priv;
1688	}
1689
1690	attach = dma_buf_attach(dmabuf, dev: dma_dev);
1691	if (IS_ERR(ptr: attach)) {
1692	err = PTR_ERR(ptr: attach);
1693	goto err_dmabuf_put;
1694	}
1695
1696	err = iio_dma_resv_lock(dmabuf, nonblock);
1697	if (err)
1698	goto err_dmabuf_detach;
1699
1700	priv->dir = buffer->direction == IIO_BUFFER_DIRECTION_IN
1701	? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1702
1703	priv->sgt = dma_buf_map_attachment(attach, priv->dir);
1704	if (IS_ERR(ptr: priv->sgt)) {
1705	err = PTR_ERR(ptr: priv->sgt);
1706	dev_err(&indio_dev->dev, "Unable to map attachment: %d\n", err);
1707	goto err_resv_unlock;
1708	}
1709
1710	kref_init(kref: &priv->ref);
1711	priv->buffer = buffer;
1712	priv->attach = attach;
1713	attach->importer_priv = priv;
1714
1715	priv->block = buffer->access->attach_dmabuf(buffer, attach);
1716	if (IS_ERR(ptr: priv->block)) {
1717	err = PTR_ERR(ptr: priv->block);
1718	goto err_dmabuf_unmap_attachment;
1719	}
1720
1721	dma_resv_unlock(obj: dmabuf->resv);
1722
1723	mutex_lock(&buffer->dmabufs_mutex);
1724
1725	/*
1726	* Check whether we already have an attachment for this driver/DMABUF
1727	* combo. If we do, refuse to attach.
1728	*/
1729	list_for_each_entry(each, &buffer->dmabufs, entry) {
1730	if (each->attach->dev == dma_dev
1731	&& each->attach->dmabuf == dmabuf) {
1732	/*
1733	* We unlocked the reservation object, so going through
1734	* the cleanup code would mean re-locking it first.
1735	* At this stage it is simpler to free the attachment
1736	* using iio_buffer_dma_put().
1737	*/
1738	mutex_unlock(lock: &buffer->dmabufs_mutex);
1739	iio_buffer_dmabuf_put(attach);
1740	return -EBUSY;
1741	}
1742	}
1743
1744	/* Otherwise, add the new attachment to our dmabufs list. */
1745	list_add(new: &priv->entry, head: &buffer->dmabufs);
1746	mutex_unlock(lock: &buffer->dmabufs_mutex);
1747
1748	return 0;
1749
1750	err_dmabuf_unmap_attachment:
1751	dma_buf_unmap_attachment(attach, priv->sgt, priv->dir);
1752	err_resv_unlock:
1753	dma_resv_unlock(obj: dmabuf->resv);
1754	err_dmabuf_detach:
1755	dma_buf_detach(dmabuf, attach);
1756	err_dmabuf_put:
1757	dma_buf_put(dmabuf);
1758	err_free_priv:
1759	kfree(objp: priv);
1760
1761	return err;
1762	}
1763
1764	static int iio_buffer_detach_dmabuf(struct iio_dev_buffer_pair *ib,
1765	int __user *user_req, bool nonblock)
1766	{
1767	struct iio_buffer *buffer = ib->buffer;
1768	struct iio_dev *indio_dev = ib->indio_dev;
1769	struct device *dma_dev = iio_buffer_get_dma_dev(indio_dev, buffer);
1770	struct iio_dmabuf_priv *priv;
1771	struct dma_buf *dmabuf;
1772	int dmabuf_fd, ret = -EPERM;
1773
1774	if (copy_from_user(to: &dmabuf_fd, from: user_req, n: sizeof(dmabuf_fd)))
1775	return -EFAULT;
1776
1777	dmabuf = dma_buf_get(fd: dmabuf_fd);
1778	if (IS_ERR(ptr: dmabuf))
1779	return PTR_ERR(ptr: dmabuf);
1780
1781	guard(mutex)(T: &buffer->dmabufs_mutex);
1782
1783	list_for_each_entry(priv, &buffer->dmabufs, entry) {
1784	if (priv->attach->dev == dma_dev
1785	&& priv->attach->dmabuf == dmabuf) {
1786	list_del(entry: &priv->entry);
1787
1788	/* Unref the reference from iio_buffer_attach_dmabuf() */
1789	iio_buffer_dmabuf_put(attach: priv->attach);
1790	ret = 0;
1791	break;
1792	}
1793	}
1794
1795	dma_buf_put(dmabuf);
1796
1797	return ret;
1798	}
1799
1800	static const char *
1801	iio_buffer_dma_fence_get_driver_name(struct dma_fence *fence)
1802	{
1803	return "iio";
1804	}
1805
1806	static void iio_buffer_dma_fence_release(struct dma_fence *fence)
1807	{
1808	struct iio_dma_fence *iio_fence =
1809	container_of(fence, struct iio_dma_fence, base);
1810
1811	kfree(objp: iio_fence);
1812	}
1813
1814	static const struct dma_fence_ops iio_buffer_dma_fence_ops = {
1815	.get_driver_name = iio_buffer_dma_fence_get_driver_name,
1816	.get_timeline_name = iio_buffer_dma_fence_get_driver_name,
1817	.release = iio_buffer_dma_fence_release,
1818	};
1819
1820	static int iio_buffer_enqueue_dmabuf(struct iio_dev_buffer_pair *ib,
1821	struct iio_dmabuf __user *iio_dmabuf_req,
1822	bool nonblock)
1823	{
1824	struct iio_buffer *buffer = ib->buffer;
1825	struct iio_dmabuf iio_dmabuf;
1826	struct dma_buf_attachment *attach;
1827	struct iio_dmabuf_priv *priv;
1828	struct iio_dma_fence *fence;
1829	struct dma_buf *dmabuf;
1830	unsigned long timeout;
1831	bool cookie, cyclic, dma_to_ram;
1832	long retl;
1833	u32 seqno;
1834	int ret;
1835
1836	if (copy_from_user(to: &iio_dmabuf, from: iio_dmabuf_req, n: sizeof(iio_dmabuf)))
1837	return -EFAULT;
1838
1839	if (iio_dmabuf.flags & ~IIO_BUFFER_DMABUF_SUPPORTED_FLAGS)
1840	return -EINVAL;
1841
1842	cyclic = iio_dmabuf.flags & IIO_BUFFER_DMABUF_CYCLIC;
1843
1844	/* Cyclic flag is only supported on output buffers */
1845	if (cyclic && buffer->direction != IIO_BUFFER_DIRECTION_OUT)
1846	return -EINVAL;
1847
1848	dmabuf = dma_buf_get(fd: iio_dmabuf.fd);
1849	if (IS_ERR(ptr: dmabuf))
1850	return PTR_ERR(ptr: dmabuf);
1851
1852	if (!iio_dmabuf.bytes_used || iio_dmabuf.bytes_used > dmabuf->size) {
1853	ret = -EINVAL;
1854	goto err_dmabuf_put;
1855	}
1856
1857	attach = iio_buffer_find_attachment(ib, dmabuf, nonblock);
1858	if (IS_ERR(ptr: attach)) {
1859	ret = PTR_ERR(ptr: attach);
1860	goto err_dmabuf_put;
1861	}
1862
1863	priv = attach->importer_priv;
1864
1865	fence = kmalloc(sizeof(*fence), GFP_KERNEL);
1866	if (!fence) {
1867	ret = -ENOMEM;
1868	goto err_attachment_put;
1869	}
1870
1871	fence->priv = priv;
1872
1873	seqno = atomic_add_return(i: 1, v: &priv->seqno);
1874
1875	/*
1876	* The transfers are guaranteed to be processed in the order they are
1877	* enqueued, so we can use a simple incrementing sequence number for
1878	* the dma_fence.
1879	*/
1880	dma_fence_init(fence: &fence->base, ops: &iio_buffer_dma_fence_ops,
1881	lock: &priv->lock, context: priv->context, seqno);
1882
1883	ret = iio_dma_resv_lock(dmabuf, nonblock);
1884	if (ret)
1885	goto err_fence_put;
1886
1887	timeout = nonblock ? 0 : msecs_to_jiffies(DMABUF_ENQUEUE_TIMEOUT_MS);
1888	dma_to_ram = buffer->direction == IIO_BUFFER_DIRECTION_IN;
1889
1890	/* Make sure we don't have writers */
1891	retl = dma_resv_wait_timeout(obj: dmabuf->resv,
1892	usage: dma_resv_usage_rw(write: dma_to_ram),
1893	intr: true, timeout);
1894	if (retl == 0)
1895	retl = -EBUSY;
1896	if (retl < 0) {
1897	ret = (int)retl;
1898	goto err_resv_unlock;
1899	}
1900
1901	if (buffer->access->lock_queue)
1902	buffer->access->lock_queue(buffer);
1903
1904	ret = dma_resv_reserve_fences(obj: dmabuf->resv, num_fences: 1);
1905	if (ret)
1906	goto err_queue_unlock;
1907
1908	dma_resv_add_fence(obj: dmabuf->resv, fence: &fence->base,
1909	usage: dma_to_ram ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ);
1910	dma_resv_unlock(obj: dmabuf->resv);
1911
1912	cookie = dma_fence_begin_signalling();
1913
1914	ret = buffer->access->enqueue_dmabuf(buffer, priv->block, &fence->base,
1915	priv->sgt, iio_dmabuf.bytes_used,
1916	cyclic);
1917	if (ret) {
1918	/*
1919	* DMABUF enqueue failed, but we already added the fence.
1920	* Signal the error through the fence completion mechanism.
1921	*/
1922	iio_buffer_signal_dmabuf_done(fence: &fence->base, ret);
1923	}
1924
1925	if (buffer->access->unlock_queue)
1926	buffer->access->unlock_queue(buffer);
1927
1928	dma_fence_end_signalling(cookie);
1929	dma_buf_put(dmabuf);
1930
1931	return ret;
1932
1933	err_queue_unlock:
1934	if (buffer->access->unlock_queue)
1935	buffer->access->unlock_queue(buffer);
1936	err_resv_unlock:
1937	dma_resv_unlock(obj: dmabuf->resv);
1938	err_fence_put:
1939	dma_fence_put(fence: &fence->base);
1940	err_attachment_put:
1941	iio_buffer_dmabuf_put(attach);
1942	err_dmabuf_put:
1943	dma_buf_put(dmabuf);
1944
1945	return ret;
1946	}
1947
1948	static void iio_buffer_cleanup(struct work_struct *work)
1949	{
1950	struct iio_dma_fence *fence =
1951	container_of(work, struct iio_dma_fence, work);
1952	struct iio_dmabuf_priv *priv = fence->priv;
1953	struct dma_buf_attachment *attach = priv->attach;
1954
1955	dma_fence_put(fence: &fence->base);
1956	iio_buffer_dmabuf_put(attach);
1957	}
1958
1959	void iio_buffer_signal_dmabuf_done(struct dma_fence *fence, int ret)
1960	{
1961	struct iio_dma_fence *iio_fence =
1962	container_of(fence, struct iio_dma_fence, base);
1963	bool cookie = dma_fence_begin_signalling();
1964
1965	/*
1966	* Get a reference to the fence, so that it's not freed as soon as
1967	* it's signaled.
1968	*/
1969	dma_fence_get(fence);
1970
1971	fence->error = ret;
1972	dma_fence_signal(fence);
1973	dma_fence_end_signalling(cookie);
1974
1975	/*
1976	* The fence will be unref'd in iio_buffer_cleanup.
1977	* It can't be done here, as the unref functions might try to lock the
1978	* resv object, which can deadlock.
1979	*/
1980	INIT_WORK(&iio_fence->work, iio_buffer_cleanup);
1981	schedule_work(work: &iio_fence->work);
1982	}
1983	EXPORT_SYMBOL_GPL(iio_buffer_signal_dmabuf_done);
1984
1985	static long iio_buffer_chrdev_ioctl(struct file *filp,
1986	unsigned int cmd, unsigned long arg)
1987	{
1988	struct iio_dev_buffer_pair *ib = filp->private_data;
1989	void __user *_arg = (void __user *)arg;
1990	bool nonblock = filp->f_flags & O_NONBLOCK;
1991
1992	switch (cmd) {
1993	case IIO_BUFFER_DMABUF_ATTACH_IOCTL:
1994	return iio_buffer_attach_dmabuf(ib, user_fd: _arg, nonblock);
1995	case IIO_BUFFER_DMABUF_DETACH_IOCTL:
1996	return iio_buffer_detach_dmabuf(ib, user_req: _arg, nonblock);
1997	case IIO_BUFFER_DMABUF_ENQUEUE_IOCTL:
1998	return iio_buffer_enqueue_dmabuf(ib, iio_dmabuf_req: _arg, nonblock);
1999	default:
2000	return -EINVAL;
2001	}
2002	}
2003
2004	static const struct file_operations iio_buffer_chrdev_fileops = {
2005	.owner = THIS_MODULE,
2006	.llseek = noop_llseek,
2007	.read = iio_buffer_read,
2008	.write = iio_buffer_write,
2009	.unlocked_ioctl = iio_buffer_chrdev_ioctl,
2010	.compat_ioctl = compat_ptr_ioctl,
2011	.poll = iio_buffer_poll,
2012	.release = iio_buffer_chrdev_release,
2013	};
2014
2015	static long iio_device_buffer_getfd(struct iio_dev *indio_dev, unsigned long arg)
2016	{
2017	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
2018	int __user *ival = (int __user *)arg;
2019	struct iio_dev_buffer_pair *ib;
2020	struct iio_buffer *buffer;
2021	int fd, idx, ret;
2022
2023	if (copy_from_user(to: &idx, from: ival, n: sizeof(idx)))
2024	return -EFAULT;
2025
2026	if (idx >= iio_dev_opaque->attached_buffers_cnt)
2027	return -ENODEV;
2028
2029	iio_device_get(indio_dev);
2030
2031	buffer = iio_dev_opaque->attached_buffers[idx];
2032
2033	if (test_and_set_bit(IIO_BUSY_BIT_POS, addr: &buffer->flags)) {
2034	ret = -EBUSY;
2035	goto error_iio_dev_put;
2036	}
2037
2038	ib = kzalloc(sizeof(*ib), GFP_KERNEL);
2039	if (!ib) {
2040	ret = -ENOMEM;
2041	goto error_clear_busy_bit;
2042	}
2043
2044	ib->indio_dev = indio_dev;
2045	ib->buffer = buffer;
2046
2047	fd = anon_inode_getfd(name: "iio:buffer", fops: &iio_buffer_chrdev_fileops,
2048	priv: ib, O_RDWR | O_CLOEXEC);
2049	if (fd < 0) {
2050	ret = fd;
2051	goto error_free_ib;
2052	}
2053
2054	if (copy_to_user(to: ival, from: &fd, n: sizeof(fd))) {
2055	/*
2056	* "Leak" the fd, as there's not much we can do about this
2057	* anyway. 'fd' might have been closed already, as
2058	* anon_inode_getfd() called fd_install() on it, which made
2059	* it reachable by userland.
2060	*
2061	* Instead of allowing a malicious user to play tricks with
2062	* us, rely on the process exit path to do any necessary
2063	* cleanup, as in releasing the file, if still needed.
2064	*/
2065	return -EFAULT;
2066	}
2067
2068	return 0;
2069
2070	error_free_ib:
2071	kfree(objp: ib);
2072	error_clear_busy_bit:
2073	clear_bit(IIO_BUSY_BIT_POS, addr: &buffer->flags);
2074	error_iio_dev_put:
2075	iio_device_put(indio_dev);
2076	return ret;
2077	}
2078
2079	static long iio_device_buffer_ioctl(struct iio_dev *indio_dev, struct file *filp,
2080	unsigned int cmd, unsigned long arg)
2081	{
2082	switch (cmd) {
2083	case IIO_BUFFER_GET_FD_IOCTL:
2084	return iio_device_buffer_getfd(indio_dev, arg);
2085	default:
2086	return IIO_IOCTL_UNHANDLED;
2087	}
2088	}
2089
2090	static int iio_channel_validate_scan_type(struct device *dev, int ch,
2091	const struct iio_scan_type *scan_type)
2092	{
2093	/* Verify that sample bits fit into storage */
2094	if (scan_type->storagebits < scan_type->realbits + scan_type->shift) {
2095	dev_err(dev,
2096	"Channel %d storagebits (%d) < shifted realbits (%d + %d)\n",
2097	ch, scan_type->storagebits,
2098	scan_type->realbits,
2099	scan_type->shift);
2100	return -EINVAL;
2101	}
2102
2103	return 0;
2104	}
2105
2106	static int __iio_buffer_alloc_sysfs_and_mask(struct iio_buffer *buffer,
2107	struct iio_dev *indio_dev,
2108	int index)
2109	{
2110	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
2111	unsigned int masklength = iio_get_masklength(indio_dev);
2112	struct iio_dev_attr *p;
2113	const struct iio_dev_attr *id_attr;
2114	struct attribute **attr;
2115	int ret, i, attrn, scan_el_attrcount, buffer_attrcount;
2116	const struct iio_chan_spec *channels;
2117
2118	buffer_attrcount = 0;
2119	if (buffer->attrs) {
2120	while (buffer->attrs[buffer_attrcount])
2121	buffer_attrcount++;
2122	}
2123	buffer_attrcount += ARRAY_SIZE(iio_buffer_attrs);
2124
2125	scan_el_attrcount = 0;
2126	INIT_LIST_HEAD(list: &buffer->buffer_attr_list);
2127	channels = indio_dev->channels;
2128	if (channels) {
2129	/* new magic */
2130	for (i = 0; i < indio_dev->num_channels; i++) {
2131	const struct iio_scan_type *scan_type;
2132
2133	if (channels[i].scan_index < 0)
2134	continue;
2135
2136	if (channels[i].has_ext_scan_type) {
2137	int j;
2138
2139	/*
2140	* get_current_scan_type is required when using
2141	* extended scan types.
2142	*/
2143	if (!indio_dev->info->get_current_scan_type) {
2144	ret = -EINVAL;
2145	goto error_cleanup_dynamic;
2146	}
2147
2148	for (j = 0; j < channels[i].num_ext_scan_type; j++) {
2149	scan_type = &channels[i].ext_scan_type[j];
2150
2151	ret = iio_channel_validate_scan_type(
2152	dev: &indio_dev->dev, ch: i, scan_type);
2153	if (ret)
2154	goto error_cleanup_dynamic;
2155	}
2156	} else {
2157	scan_type = &channels[i].scan_type;
2158
2159	ret = iio_channel_validate_scan_type(
2160	dev: &indio_dev->dev, ch: i, scan_type);
2161	if (ret)
2162	goto error_cleanup_dynamic;
2163	}
2164
2165	ret = iio_buffer_add_channel_sysfs(indio_dev, buffer,
2166	chan: &channels[i]);
2167	if (ret < 0)
2168	goto error_cleanup_dynamic;
2169	scan_el_attrcount += ret;
2170	if (channels[i].type == IIO_TIMESTAMP)
2171	iio_dev_opaque->scan_index_timestamp =
2172	channels[i].scan_index;
2173	}
2174	if (masklength && !buffer->scan_mask) {
2175	buffer->scan_mask = bitmap_zalloc(nbits: masklength,
2176	GFP_KERNEL);
2177	if (!buffer->scan_mask) {
2178	ret = -ENOMEM;
2179	goto error_cleanup_dynamic;
2180	}
2181	}
2182	}
2183
2184	attrn = buffer_attrcount + scan_el_attrcount;
2185	attr = kcalloc(attrn + 1, sizeof(*attr), GFP_KERNEL);
2186	if (!attr) {
2187	ret = -ENOMEM;
2188	goto error_free_scan_mask;
2189	}
2190
2191	memcpy(attr, iio_buffer_attrs, sizeof(iio_buffer_attrs));
2192	if (!buffer->access->set_length)
2193	attr[0] = &dev_attr_length_ro.attr;
2194
2195	if (buffer->access->flags & INDIO_BUFFER_FLAG_FIXED_WATERMARK)
2196	attr[2] = &dev_attr_watermark_ro.attr;
2197
2198	if (buffer->attrs)
2199	for (i = 0, id_attr = buffer->attrs[i];
2200	(id_attr = buffer->attrs[i]); i++)
2201	attr[ARRAY_SIZE(iio_buffer_attrs) + i] =
2202	(struct attribute *)&id_attr->dev_attr.attr;
2203
2204	buffer->buffer_group.attrs = attr;
2205
2206	for (i = 0; i < buffer_attrcount; i++) {
2207	struct attribute *wrapped;
2208
2209	wrapped = iio_buffer_wrap_attr(buffer, attr: attr[i]);
2210	if (!wrapped) {
2211	ret = -ENOMEM;
2212	goto error_free_buffer_attrs;
2213	}
2214	attr[i] = wrapped;
2215	}
2216
2217	attrn = 0;
2218	list_for_each_entry(p, &buffer->buffer_attr_list, l)
2219	attr[attrn++] = &p->dev_attr.attr;
2220
2221	buffer->buffer_group.name = kasprintf(GFP_KERNEL, fmt: "buffer%d", index);
2222	if (!buffer->buffer_group.name) {
2223	ret = -ENOMEM;
2224	goto error_free_buffer_attrs;
2225	}
2226
2227	ret = iio_device_register_sysfs_group(indio_dev, group: &buffer->buffer_group);
2228	if (ret)
2229	goto error_free_buffer_attr_group_name;
2230
2231	/* we only need to register the legacy groups for the first buffer */
2232	if (index > 0)
2233	return 0;
2234
2235	ret = iio_buffer_register_legacy_sysfs_groups(indio_dev, buffer_attrs: attr,
2236	buffer_attrcount,
2237	scan_el_attrcount);
2238	if (ret)
2239	goto error_free_buffer_attr_group_name;
2240
2241	return 0;
2242
2243	error_free_buffer_attr_group_name:
2244	kfree(objp: buffer->buffer_group.name);
2245	error_free_buffer_attrs:
2246	kfree(objp: buffer->buffer_group.attrs);
2247	error_free_scan_mask:
2248	bitmap_free(bitmap: buffer->scan_mask);
2249	error_cleanup_dynamic:
2250	iio_free_chan_devattr_list(attr_list: &buffer->buffer_attr_list);
2251
2252	return ret;
2253	}
2254
2255	static void __iio_buffer_free_sysfs_and_mask(struct iio_buffer *buffer,
2256	struct iio_dev *indio_dev,
2257	int index)
2258	{
2259	if (index == 0)
2260	iio_buffer_unregister_legacy_sysfs_groups(indio_dev);
2261	bitmap_free(bitmap: buffer->scan_mask);
2262	kfree(objp: buffer->buffer_group.name);
2263	kfree(objp: buffer->buffer_group.attrs);
2264	iio_free_chan_devattr_list(attr_list: &buffer->buffer_attr_list);
2265	}
2266
2267	int iio_buffers_alloc_sysfs_and_mask(struct iio_dev *indio_dev)
2268	{
2269	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
2270	const struct iio_chan_spec *channels;
2271	struct iio_buffer *buffer;
2272	int ret, i, idx;
2273	size_t sz;
2274
2275	channels = indio_dev->channels;
2276	if (channels) {
2277	int ml = 0;
2278
2279	for (i = 0; i < indio_dev->num_channels; i++)
2280	ml = max(ml, channels[i].scan_index + 1);
2281	ACCESS_PRIVATE(indio_dev, masklength) = ml;
2282	}
2283
2284	if (!iio_dev_opaque->attached_buffers_cnt)
2285	return 0;
2286
2287	for (idx = 0; idx < iio_dev_opaque->attached_buffers_cnt; idx++) {
2288	buffer = iio_dev_opaque->attached_buffers[idx];
2289	ret = __iio_buffer_alloc_sysfs_and_mask(buffer, indio_dev, index: idx);
2290	if (ret)
2291	goto error_unwind_sysfs_and_mask;
2292	}
2293
2294	sz = sizeof(*iio_dev_opaque->buffer_ioctl_handler);
2295	iio_dev_opaque->buffer_ioctl_handler = kzalloc(sz, GFP_KERNEL);
2296	if (!iio_dev_opaque->buffer_ioctl_handler) {
2297	ret = -ENOMEM;
2298	goto error_unwind_sysfs_and_mask;
2299	}
2300
2301	iio_dev_opaque->buffer_ioctl_handler->ioctl = iio_device_buffer_ioctl;
2302	iio_device_ioctl_handler_register(indio_dev,
2303	h: iio_dev_opaque->buffer_ioctl_handler);
2304
2305	return 0;
2306
2307	error_unwind_sysfs_and_mask:
2308	while (idx--) {
2309	buffer = iio_dev_opaque->attached_buffers[idx];
2310	__iio_buffer_free_sysfs_and_mask(buffer, indio_dev, index: idx);
2311	}
2312	return ret;
2313	}
2314
2315	void iio_buffers_free_sysfs_and_mask(struct iio_dev *indio_dev)
2316	{
2317	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
2318	struct iio_buffer *buffer;
2319	int i;
2320
2321	if (!iio_dev_opaque->attached_buffers_cnt)
2322	return;
2323
2324	iio_device_ioctl_handler_unregister(h: iio_dev_opaque->buffer_ioctl_handler);
2325	kfree(objp: iio_dev_opaque->buffer_ioctl_handler);
2326
2327	for (i = iio_dev_opaque->attached_buffers_cnt - 1; i >= 0; i--) {
2328	buffer = iio_dev_opaque->attached_buffers[i];
2329	__iio_buffer_free_sysfs_and_mask(buffer, indio_dev, index: i);
2330	}
2331	}
2332
2333	/**
2334	* iio_validate_scan_mask_onehot() - Validates that exactly one channel is selected
2335	* @indio_dev: the iio device
2336	* @mask: scan mask to be checked
2337	*
2338	* Return true if exactly one bit is set in the scan mask, false otherwise. It
2339	* can be used for devices where only one channel can be active for sampling at
2340	* a time.
2341	*/
2342	bool iio_validate_scan_mask_onehot(struct iio_dev *indio_dev,
2343	const unsigned long *mask)
2344	{
2345	return bitmap_weight(src: mask, nbits: iio_get_masklength(indio_dev)) == 1;
2346	}
2347	EXPORT_SYMBOL_GPL(iio_validate_scan_mask_onehot);
2348
2349	static const void *iio_demux(struct iio_buffer *buffer,
2350	const void *datain)
2351	{
2352	struct iio_demux_table *t;
2353
2354	if (list_empty(head: &buffer->demux_list))
2355	return datain;
2356	list_for_each_entry(t, &buffer->demux_list, l)
2357	memcpy(buffer->demux_bounce + t->to,
2358	datain + t->from, t->length);
2359
2360	return buffer->demux_bounce;
2361	}
2362
2363	static int iio_push_to_buffer(struct iio_buffer *buffer, const void *data)
2364	{
2365	const void *dataout = iio_demux(buffer, datain: data);
2366	int ret;
2367
2368	ret = buffer->access->store_to(buffer, dataout);
2369	if (ret)
2370	return ret;
2371
2372	/*
2373	* We can't just test for watermark to decide if we wake the poll queue
2374	* because read may request less samples than the watermark.
2375	*/
2376	wake_up_interruptible_poll(&buffer->pollq, EPOLLIN | EPOLLRDNORM);
2377	return 0;
2378	}
2379
2380	/**
2381	* iio_push_to_buffers() - push to a registered buffer.
2382	* @indio_dev: iio_dev structure for device.
2383	* @data: Full scan.
2384	*
2385	* Context: Any context.
2386	* Return: 0 on success, negative error code on failure.
2387	*/
2388	int iio_push_to_buffers(struct iio_dev *indio_dev, const void *data)
2389	{
2390	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
2391	int ret;
2392	struct iio_buffer *buf;
2393
2394	list_for_each_entry(buf, &iio_dev_opaque->buffer_list, buffer_list) {
2395	ret = iio_push_to_buffer(buffer: buf, data);
2396	if (ret < 0)
2397	return ret;
2398	}
2399
2400	return 0;
2401	}
2402	EXPORT_SYMBOL_GPL(iio_push_to_buffers);
2403
2404	/**
2405	* iio_push_to_buffers_with_ts_unaligned() - push to registered buffer,
2406	* no alignment or space requirements.
2407	* @indio_dev: iio_dev structure for device.
2408	* @data: channel data excluding the timestamp.
2409	* @data_sz: size of data.
2410	* @timestamp: timestamp for the sample data.
2411	*
2412	* This special variant of iio_push_to_buffers_with_timestamp() does
2413	* not require space for the timestamp, or 8 byte alignment of data.
2414	* It does however require an allocation on first call and additional
2415	* copies on all calls, so should be avoided if possible.
2416	*
2417	* Context: May sleep.
2418	* Return: 0 on success, negative error code on failure.
2419	*/
2420	int iio_push_to_buffers_with_ts_unaligned(struct iio_dev *indio_dev,
2421	const void *data,
2422	size_t data_sz,
2423	int64_t timestamp)
2424	{
2425	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
2426
2427	might_sleep();
2428
2429	/*
2430	* Conservative estimate - we can always safely copy the minimum
2431	* of either the data provided or the length of the destination buffer.
2432	* This relaxed limit allows the calling drivers to be lax about
2433	* tracking the size of the data they are pushing, at the cost of
2434	* unnecessary copying of padding.
2435	*/
2436	data_sz = min_t(size_t, indio_dev->scan_bytes, data_sz);
2437	if (iio_dev_opaque->bounce_buffer_size != indio_dev->scan_bytes) {
2438	void *bb;
2439
2440	bb = devm_krealloc(dev: &indio_dev->dev,
2441	ptr: iio_dev_opaque->bounce_buffer,
2442	size: indio_dev->scan_bytes, GFP_KERNEL);
2443	if (!bb)
2444	return -ENOMEM;
2445	iio_dev_opaque->bounce_buffer = bb;
2446	iio_dev_opaque->bounce_buffer_size = indio_dev->scan_bytes;
2447	}
2448	memcpy(iio_dev_opaque->bounce_buffer, data, data_sz);
2449	return iio_push_to_buffers_with_timestamp(indio_dev,
2450	data: iio_dev_opaque->bounce_buffer,
2451	timestamp);
2452	}
2453	EXPORT_SYMBOL_GPL(iio_push_to_buffers_with_ts_unaligned);
2454
2455	/**
2456	* iio_buffer_release() - Free a buffer's resources
2457	* @ref: Pointer to the kref embedded in the iio_buffer struct
2458	*
2459	* This function is called when the last reference to the buffer has been
2460	* dropped. It will typically free all resources allocated by the buffer. Do not
2461	* call this function manually, always use iio_buffer_put() when done using a
2462	* buffer.
2463	*/
2464	static void iio_buffer_release(struct kref *ref)
2465	{
2466	struct iio_buffer *buffer = container_of(ref, struct iio_buffer, ref);
2467
2468	mutex_destroy(lock: &buffer->dmabufs_mutex);
2469	buffer->access->release(buffer);
2470	}
2471
2472	/**
2473	* iio_buffer_get() - Grab a reference to the buffer
2474	* @buffer: The buffer to grab a reference for, may be NULL
2475	*
2476	* Returns the pointer to the buffer that was passed into the function.
2477	*/
2478	struct iio_buffer *iio_buffer_get(struct iio_buffer *buffer)
2479	{
2480	if (buffer)
2481	kref_get(kref: &buffer->ref);
2482
2483	return buffer;
2484	}
2485	EXPORT_SYMBOL_GPL(iio_buffer_get);
2486
2487	/**
2488	* iio_buffer_put() - Release the reference to the buffer
2489	* @buffer: The buffer to release the reference for, may be NULL
2490	*/
2491	void iio_buffer_put(struct iio_buffer *buffer)
2492	{
2493	if (buffer)
2494	kref_put(kref: &buffer->ref, release: iio_buffer_release);
2495	}
2496	EXPORT_SYMBOL_GPL(iio_buffer_put);
2497
2498	/**
2499	* iio_device_attach_buffer - Attach a buffer to a IIO device
2500	* @indio_dev: The device the buffer should be attached to
2501	* @buffer: The buffer to attach to the device
2502	*
2503	* Return 0 if successful, negative if error.
2504	*
2505	* This function attaches a buffer to a IIO device. The buffer stays attached to
2506	* the device until the device is freed. For legacy reasons, the first attached
2507	* buffer will also be assigned to 'indio_dev->buffer'.
2508	* The array allocated here, will be free'd via the iio_device_detach_buffers()
2509	* call which is handled by the iio_device_free().
2510	*/
2511	int iio_device_attach_buffer(struct iio_dev *indio_dev,
2512	struct iio_buffer *buffer)
2513	{
2514	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
2515	struct iio_buffer **new, **old = iio_dev_opaque->attached_buffers;
2516	unsigned int cnt = iio_dev_opaque->attached_buffers_cnt;
2517
2518	cnt++;
2519
2520	new = krealloc(old, sizeof(*new) * cnt, GFP_KERNEL);
2521	if (!new)
2522	return -ENOMEM;
2523	iio_dev_opaque->attached_buffers = new;
2524
2525	buffer = iio_buffer_get(buffer);
2526
2527	/* first buffer is legacy; attach it to the IIO device directly */
2528	if (!indio_dev->buffer)
2529	indio_dev->buffer = buffer;
2530
2531	iio_dev_opaque->attached_buffers[cnt - 1] = buffer;
2532	iio_dev_opaque->attached_buffers_cnt = cnt;
2533
2534	return 0;
2535	}
2536	EXPORT_SYMBOL_GPL(iio_device_attach_buffer);
2537