1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Block driver for media (i.e., flash cards)
4	*
5	* Copyright 2002 Hewlett-Packard Company
6	* Copyright 2005-2008 Pierre Ossman
7	*
8	* Use consistent with the GNU GPL is permitted,
9	* provided that this copyright notice is
10	* preserved in its entirety in all copies and derived works.
11	*
12	* HEWLETT-PACKARD COMPANY MAKES NO WARRANTIES, EXPRESSED OR IMPLIED,
13	* AS TO THE USEFULNESS OR CORRECTNESS OF THIS CODE OR ITS
14	* FITNESS FOR ANY PARTICULAR PURPOSE.
15	*
16	* Many thanks to Alessandro Rubini and Jonathan Corbet!
17	*
18	* Author: Andrew Christian
19	* 28 May 2002
20	*/
21	#include <linux/moduleparam.h>
22	#include <linux/module.h>
23	#include <linux/init.h>
24
25	#include <linux/kernel.h>
26	#include <linux/fs.h>
27	#include <linux/slab.h>
28	#include <linux/errno.h>
29	#include <linux/hdreg.h>
30	#include <linux/kdev_t.h>
31	#include <linux/kref.h>
32	#include <linux/blkdev.h>
33	#include <linux/cdev.h>
34	#include <linux/mutex.h>
35	#include <linux/scatterlist.h>
36	#include <linux/string.h>
37	#include <linux/string_helpers.h>
38	#include <linux/delay.h>
39	#include <linux/capability.h>
40	#include <linux/compat.h>
41	#include <linux/pm_runtime.h>
42	#include <linux/idr.h>
43	#include <linux/debugfs.h>
44	#include <linux/rpmb.h>
45
46	#include <linux/mmc/ioctl.h>
47	#include <linux/mmc/card.h>
48	#include <linux/mmc/host.h>
49	#include <linux/mmc/mmc.h>
50	#include <linux/mmc/sd.h>
51
52	#include <linux/uaccess.h>
53	#include <linux/unaligned.h>
54
55	#include "queue.h"
56	#include "block.h"
57	#include "core.h"
58	#include "card.h"
59	#include "crypto.h"
60	#include "host.h"
61	#include "bus.h"
62	#include "mmc_ops.h"
63	#include "quirks.h"
64	#include "sd_ops.h"
65
66	MODULE_ALIAS("mmc:block");
67	#ifdef MODULE_PARAM_PREFIX
68	#undef MODULE_PARAM_PREFIX
69	#endif
70	#define MODULE_PARAM_PREFIX "mmcblk."
71
72	/*
73	* Set a 10 second timeout for polling write request busy state. Note, mmc core
74	* is setting a 3 second timeout for SD cards, and SDHCI has long had a 10
75	* second software timer to timeout the whole request, so 10 seconds should be
76	* ample.
77	*/
78	#define MMC_BLK_TIMEOUT_MS (10 * 1000)
79	#define MMC_EXTRACT_INDEX_FROM_ARG(x) ((x & 0x00FF0000) >> 16)
80	#define MMC_EXTRACT_VALUE_FROM_ARG(x) ((x & 0x0000FF00) >> 8)
81
82	#define RPMB_FRAME_SIZE sizeof(struct rpmb_frame)
83	#define CHECK_SIZE_NEQ(val) ((val) != sizeof(struct rpmb_frame))
84	#define CHECK_SIZE_ALIGNED(val) IS_ALIGNED((val), sizeof(struct rpmb_frame))
85
86	static DEFINE_MUTEX(block_mutex);
87
88	/*
89	* The defaults come from config options but can be overriden by module
90	* or bootarg options.
91	*/
92	static int perdev_minors = CONFIG_MMC_BLOCK_MINORS;
93
94	/*
95	* We've only got one major, so number of mmcblk devices is
96	* limited to (1 << 20) / number of minors per device. It is also
97	* limited by the MAX_DEVICES below.
98	*/
99	static int max_devices;
100
101	#define MAX_DEVICES 256
102
103	static DEFINE_IDA(mmc_blk_ida);
104	static DEFINE_IDA(mmc_rpmb_ida);
105
106	struct mmc_blk_busy_data {
107	struct mmc_card *card;
108	u32 status;
109	};
110
111	/*
112	* There is one mmc_blk_data per slot.
113	*/
114	struct mmc_blk_data {
115	struct device *parent;
116	struct gendisk *disk;
117	struct mmc_queue queue;
118	struct list_head part;
119	struct list_head rpmbs;
120
121	unsigned int flags;
122	#define MMC_BLK_CMD23 (1 << 0) /* Can do SET_BLOCK_COUNT for multiblock */
123	#define MMC_BLK_REL_WR (1 << 1) /* MMC Reliable write support */
124
125	struct kref kref;
126	unsigned int read_only;
127	unsigned int part_type;
128	unsigned int reset_done;
129	#define MMC_BLK_READ BIT(0)
130	#define MMC_BLK_WRITE BIT(1)
131	#define MMC_BLK_DISCARD BIT(2)
132	#define MMC_BLK_SECDISCARD BIT(3)
133	#define MMC_BLK_CQE_RECOVERY BIT(4)
134	#define MMC_BLK_TRIM BIT(5)
135
136	/*
137	* Only set in main mmc_blk_data associated
138	* with mmc_card with dev_set_drvdata, and keeps
139	* track of the current selected device partition.
140	*/
141	unsigned int part_curr;
142	#define MMC_BLK_PART_INVALID UINT_MAX /* Unknown partition active */
143	int area_type;
144
145	/* debugfs files (only in main mmc_blk_data) */
146	struct dentry *status_dentry;
147	struct dentry *ext_csd_dentry;
148	};
149
150	/* Device type for RPMB character devices */
151	static dev_t mmc_rpmb_devt;
152
153	/* Bus type for RPMB character devices */
154	static const struct bus_type mmc_rpmb_bus_type = {
155	.name = "mmc_rpmb",
156	};
157
158	/**
159	* struct mmc_rpmb_data - special RPMB device type for these areas
160	* @dev: the device for the RPMB area
161	* @chrdev: character device for the RPMB area
162	* @id: unique device ID number
163	* @part_index: partition index (0 on first)
164	* @md: parent MMC block device
165	* @rdev: registered RPMB device
166	* @node: list item, so we can put this device on a list
167	*/
168	struct mmc_rpmb_data {
169	struct device dev;
170	struct cdev chrdev;
171	int id;
172	unsigned int part_index;
173	struct mmc_blk_data *md;
174	struct rpmb_dev *rdev;
175	struct list_head node;
176	};
177
178	static DEFINE_MUTEX(open_lock);
179
180	module_param(perdev_minors, int, 0444);
181	MODULE_PARM_DESC(perdev_minors, "Minors numbers to allocate per device");
182
183	static inline int mmc_blk_part_switch(struct mmc_card *card,
184	unsigned int part_type);
185	static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
186	struct mmc_card *card,
187	int recovery_mode,
188	struct mmc_queue *mq);
189	static void mmc_blk_hsq_req_done(struct mmc_request *mrq);
190	static int mmc_spi_err_check(struct mmc_card *card);
191	static int mmc_blk_busy_cb(void *cb_data, bool *busy);
192
193	static struct mmc_blk_data *mmc_blk_get(struct gendisk *disk)
194	{
195	struct mmc_blk_data *md;
196
197	mutex_lock(&open_lock);
198	md = disk->private_data;
199	if (md && !kref_get_unless_zero(kref: &md->kref))
200	md = NULL;
201	mutex_unlock(lock: &open_lock);
202
203	return md;
204	}
205
206	static inline int mmc_get_devidx(struct gendisk *disk)
207	{
208	int devidx = disk->first_minor / perdev_minors;
209	return devidx;
210	}
211
212	static void mmc_blk_kref_release(struct kref *ref)
213	{
214	struct mmc_blk_data *md = container_of(ref, struct mmc_blk_data, kref);
215	int devidx;
216
217	devidx = mmc_get_devidx(disk: md->disk);
218	ida_free(&mmc_blk_ida, id: devidx);
219
220	mutex_lock(&open_lock);
221	md->disk->private_data = NULL;
222	mutex_unlock(lock: &open_lock);
223
224	put_disk(disk: md->disk);
225	kfree(objp: md);
226	}
227
228	static void mmc_blk_put(struct mmc_blk_data *md)
229	{
230	kref_put(kref: &md->kref, release: mmc_blk_kref_release);
231	}
232
233	static ssize_t power_ro_lock_show(struct device *dev,
234	struct device_attribute *attr, char *buf)
235	{
236	int ret;
237	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
238	struct mmc_card *card = md->queue.card;
239	int locked = 0;
240
241	if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PERM_WP_EN)
242	locked = 2;
243	else if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PWR_WP_EN)
244	locked = 1;
245
246	ret = sysfs_emit(buf, fmt: "%d\n", locked);
247
248	mmc_blk_put(md);
249
250	return ret;
251	}
252
253	static ssize_t power_ro_lock_store(struct device *dev,
254	struct device_attribute *attr, const char *buf, size_t count)
255	{
256	int ret;
257	struct mmc_blk_data *md, *part_md;
258	struct mmc_queue *mq;
259	struct request *req;
260	unsigned long set;
261
262	if (kstrtoul(s: buf, base: 0, res: &set))
263	return -EINVAL;
264
265	if (set != 1)
266	return count;
267
268	md = mmc_blk_get(dev_to_disk(dev));
269	mq = &md->queue;
270
271	/* Dispatch locking to the block layer */
272	req = blk_mq_alloc_request(q: mq->queue, opf: REQ_OP_DRV_OUT, flags: 0);
273	if (IS_ERR(ptr: req)) {
274	count = PTR_ERR(ptr: req);
275	goto out_put;
276	}
277	req_to_mmc_queue_req(rq: req)->drv_op = MMC_DRV_OP_BOOT_WP;
278	req_to_mmc_queue_req(rq: req)->drv_op_result = -EIO;
279	blk_execute_rq(rq: req, at_head: false);
280	ret = req_to_mmc_queue_req(rq: req)->drv_op_result;
281	blk_mq_free_request(rq: req);
282
283	if (!ret) {
284	pr_info("%s: Locking boot partition ro until next power on\n",
285	md->disk->disk_name);
286	set_disk_ro(disk: md->disk, read_only: 1);
287
288	list_for_each_entry(part_md, &md->part, part)
289	if (part_md->area_type == MMC_BLK_DATA_AREA_BOOT) {
290	pr_info("%s: Locking boot partition ro until next power on\n", part_md->disk->disk_name);
291	set_disk_ro(disk: part_md->disk, read_only: 1);
292	}
293	}
294	out_put:
295	mmc_blk_put(md);
296	return count;
297	}
298
299	static DEVICE_ATTR(ro_lock_until_next_power_on, 0,
300	power_ro_lock_show, power_ro_lock_store);
301
302	static ssize_t force_ro_show(struct device *dev, struct device_attribute *attr,
303	char *buf)
304	{
305	int ret;
306	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
307
308	ret = sysfs_emit(buf, fmt: "%d\n",
309	get_disk_ro(dev_to_disk(dev)) ^
310	md->read_only);
311	mmc_blk_put(md);
312	return ret;
313	}
314
315	static ssize_t force_ro_store(struct device *dev, struct device_attribute *attr,
316	const char *buf, size_t count)
317	{
318	int ret;
319	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
320	unsigned long set;
321
322	if (kstrtoul(s: buf, base: 0, res: &set)) {
323	ret = -EINVAL;
324	goto out;
325	}
326
327	set_disk_ro(dev_to_disk(dev), read_only: set || md->read_only);
328	ret = count;
329	out:
330	mmc_blk_put(md);
331	return ret;
332	}
333
334	static DEVICE_ATTR(force_ro, 0644, force_ro_show, force_ro_store);
335
336	static struct attribute *mmc_disk_attrs[] = {
337	&dev_attr_force_ro.attr,
338	&dev_attr_ro_lock_until_next_power_on.attr,
339	NULL,
340	};
341
342	static umode_t mmc_disk_attrs_is_visible(struct kobject *kobj,
343	struct attribute *a, int n)
344	{
345	struct device *dev = kobj_to_dev(kobj);
346	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
347	umode_t mode = a->mode;
348
349	if (a == &dev_attr_ro_lock_until_next_power_on.attr &&
350	(md->area_type & MMC_BLK_DATA_AREA_BOOT) &&
351	md->queue.card->ext_csd.boot_ro_lockable) {
352	mode = 0444;
353	if (!(md->queue.card->ext_csd.boot_ro_lock &
354	EXT_CSD_BOOT_WP_B_PWR_WP_DIS))
355	mode |= 0200;
356	}
357
358	mmc_blk_put(md);
359	return mode;
360	}
361
362	static const struct attribute_group mmc_disk_attr_group = {
363	.is_visible = mmc_disk_attrs_is_visible,
364	.attrs = mmc_disk_attrs,
365	};
366
367	static const struct attribute_group *mmc_disk_attr_groups[] = {
368	&mmc_disk_attr_group,
369	NULL,
370	};
371
372	static int mmc_blk_open(struct gendisk *disk, blk_mode_t mode)
373	{
374	struct mmc_blk_data *md = mmc_blk_get(disk);
375	int ret = -ENXIO;
376
377	mutex_lock(&block_mutex);
378	if (md) {
379	ret = 0;
380	if ((mode & BLK_OPEN_WRITE) && md->read_only) {
381	mmc_blk_put(md);
382	ret = -EROFS;
383	}
384	}
385	mutex_unlock(lock: &block_mutex);
386
387	return ret;
388	}
389
390	static void mmc_blk_release(struct gendisk *disk)
391	{
392	struct mmc_blk_data *md = disk->private_data;
393
394	mutex_lock(&block_mutex);
395	mmc_blk_put(md);
396	mutex_unlock(lock: &block_mutex);
397	}
398
399	static int
400	mmc_blk_getgeo(struct gendisk *disk, struct hd_geometry *geo)
401	{
402	geo->cylinders = get_capacity(disk) / (4 * 16);
403	geo->heads = 4;
404	geo->sectors = 16;
405	return 0;
406	}
407
408	struct mmc_blk_ioc_data {
409	struct mmc_ioc_cmd ic;
410	unsigned char *buf;
411	u64 buf_bytes;
412	unsigned int flags;
413	#define MMC_BLK_IOC_DROP BIT(0) /* drop this mrq */
414	#define MMC_BLK_IOC_SBC BIT(1) /* use mrq.sbc */
415
416	struct mmc_rpmb_data *rpmb;
417	};
418
419	static struct mmc_blk_ioc_data *mmc_blk_ioctl_copy_from_user(
420	struct mmc_ioc_cmd __user *user)
421	{
422	struct mmc_blk_ioc_data *idata;
423	int err;
424
425	idata = kzalloc(sizeof(*idata), GFP_KERNEL);
426	if (!idata) {
427	err = -ENOMEM;
428	goto out;
429	}
430
431	if (copy_from_user(to: &idata->ic, from: user, n: sizeof(idata->ic))) {
432	err = -EFAULT;
433	goto idata_err;
434	}
435
436	idata->buf_bytes = (u64) idata->ic.blksz * idata->ic.blocks;
437	if (idata->buf_bytes > MMC_IOC_MAX_BYTES) {
438	err = -EOVERFLOW;
439	goto idata_err;
440	}
441
442	if (!idata->buf_bytes) {
443	idata->buf = NULL;
444	return idata;
445	}
446
447	idata->buf = memdup_user((void __user *)(unsigned long)
448	idata->ic.data_ptr, idata->buf_bytes);
449	if (IS_ERR(ptr: idata->buf)) {
450	err = PTR_ERR(ptr: idata->buf);
451	goto idata_err;
452	}
453
454	return idata;
455
456	idata_err:
457	kfree(objp: idata);
458	out:
459	return ERR_PTR(error: err);
460	}
461
462	static int mmc_blk_ioctl_copy_to_user(struct mmc_ioc_cmd __user *ic_ptr,
463	struct mmc_blk_ioc_data *idata)
464	{
465	struct mmc_ioc_cmd *ic = &idata->ic;
466
467	if (copy_to_user(to: &(ic_ptr->response), from: ic->response,
468	n: sizeof(ic->response)))
469	return -EFAULT;
470
471	if (!idata->ic.write_flag) {
472	if (copy_to_user(to: (void __user *)(unsigned long)ic->data_ptr,
473	from: idata->buf, n: idata->buf_bytes))
474	return -EFAULT;
475	}
476
477	return 0;
478	}
479
480	static int __mmc_blk_ioctl_cmd(struct mmc_card *card, struct mmc_blk_data *md,
481	struct mmc_blk_ioc_data **idatas, int i)
482	{
483	struct mmc_command cmd = {}, sbc = {};
484	struct mmc_data data = {};
485	struct mmc_request mrq = {};
486	struct scatterlist sg;
487	bool r1b_resp;
488	unsigned int busy_timeout_ms;
489	int err;
490	unsigned int target_part;
491	struct mmc_blk_ioc_data *idata = idatas[i];
492	struct mmc_blk_ioc_data *prev_idata = NULL;
493
494	if (!card || !md || !idata)
495	return -EINVAL;
496
497	if (idata->flags & MMC_BLK_IOC_DROP)
498	return 0;
499
500	if (idata->flags & MMC_BLK_IOC_SBC && i > 0)
501	prev_idata = idatas[i - 1];
502
503	/*
504	* The RPMB accesses comes in from the character device, so we
505	* need to target these explicitly. Else we just target the
506	* partition type for the block device the ioctl() was issued
507	* on.
508	*/
509	if (idata->rpmb) {
510	/* Support multiple RPMB partitions */
511	target_part = idata->rpmb->part_index;
512	target_part |= EXT_CSD_PART_CONFIG_ACC_RPMB;
513	} else {
514	target_part = md->part_type;
515	}
516
517	cmd.opcode = idata->ic.opcode;
518	cmd.arg = idata->ic.arg;
519	cmd.flags = idata->ic.flags;
520
521	if (idata->buf_bytes) {
522	data.sg = &sg;
523	data.sg_len = 1;
524	data.blksz = idata->ic.blksz;
525	data.blocks = idata->ic.blocks;
526
527	sg_init_one(data.sg, idata->buf, idata->buf_bytes);
528
529	if (idata->ic.write_flag)
530	data.flags = MMC_DATA_WRITE;
531	else
532	data.flags = MMC_DATA_READ;
533
534	/* data.flags must already be set before doing this. */
535	mmc_set_data_timeout(data: &data, card);
536
537	/* Allow overriding the timeout_ns for empirical tuning. */
538	if (idata->ic.data_timeout_ns)
539	data.timeout_ns = idata->ic.data_timeout_ns;
540
541	mrq.data = &data;
542	}
543
544	mrq.cmd = &cmd;
545
546	err = mmc_blk_part_switch(card, part_type: target_part);
547	if (err)
548	return err;
549
550	if (idata->ic.is_acmd) {
551	err = mmc_app_cmd(host: card->host, card);
552	if (err)
553	return err;
554	}
555
556	if (idata->rpmb || prev_idata) {
557	sbc.opcode = MMC_SET_BLOCK_COUNT;
558	/*
559	* We don't do any blockcount validation because the max size
560	* may be increased by a future standard. We just copy the
561	* 'Reliable Write' bit here.
562	*/
563	sbc.arg = data.blocks | (idata->ic.write_flag & BIT(31));
564	if (prev_idata)
565	sbc.arg = prev_idata->ic.arg;
566	sbc.flags = MMC_RSP_R1 | MMC_CMD_AC;
567	mrq.sbc = &sbc;
568	}
569
570	if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_SANITIZE_START) &&
571	(cmd.opcode == MMC_SWITCH))
572	return mmc_sanitize(card, timeout_ms: idata->ic.cmd_timeout_ms);
573
574	/* If it's an R1B response we need some more preparations. */
575	busy_timeout_ms = idata->ic.cmd_timeout_ms ? : MMC_BLK_TIMEOUT_MS;
576	r1b_resp = (cmd.flags & MMC_RSP_R1B) == MMC_RSP_R1B;
577	if (r1b_resp)
578	mmc_prepare_busy_cmd(host: card->host, cmd: &cmd, timeout_ms: busy_timeout_ms);
579
580	mmc_wait_for_req(host: card->host, mrq: &mrq);
581	memcpy(&idata->ic.response, cmd.resp, sizeof(cmd.resp));
582
583	if (prev_idata) {
584	memcpy(&prev_idata->ic.response, sbc.resp, sizeof(sbc.resp));
585	if (sbc.error) {
586	dev_err(mmc_dev(card->host), "%s: sbc error %d\n",
587	__func__, sbc.error);
588	return sbc.error;
589	}
590	}
591
592	if (cmd.error) {
593	dev_err(mmc_dev(card->host), "%s: cmd error %d\n",
594	__func__, cmd.error);
595	return cmd.error;
596	}
597	if (data.error) {
598	dev_err(mmc_dev(card->host), "%s: data error %d\n",
599	__func__, data.error);
600	return data.error;
601	}
602
603	/*
604	* Make sure the cache of the PARTITION_CONFIG register and
605	* PARTITION_ACCESS bits is updated in case the ioctl ext_csd write
606	* changed it successfully.
607	*/
608	if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_PART_CONFIG) &&
609	(cmd.opcode == MMC_SWITCH)) {
610	struct mmc_blk_data *main_md = dev_get_drvdata(dev: &card->dev);
611	u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg);
612
613	/*
614	* Update cache so the next mmc_blk_part_switch call operates
615	* on up-to-date data.
616	*/
617	card->ext_csd.part_config = value;
618	main_md->part_curr = value & EXT_CSD_PART_CONFIG_ACC_MASK;
619	}
620
621	/*
622	* Make sure to update CACHE_CTRL in case it was changed. The cache
623	* will get turned back on if the card is re-initialized, e.g.
624	* suspend/resume or hw reset in recovery.
625	*/
626	if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_CACHE_CTRL) &&
627	(cmd.opcode == MMC_SWITCH)) {
628	u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg) & 1;
629
630	card->ext_csd.cache_ctrl = value;
631	}
632
633	/*
634	* According to the SD specs, some commands require a delay after
635	* issuing the command.
636	*/
637	if (idata->ic.postsleep_min_us)
638	usleep_range(min: idata->ic.postsleep_min_us, max: idata->ic.postsleep_max_us);
639
640	if (mmc_host_is_spi(card->host)) {
641	if (idata->ic.write_flag || r1b_resp || cmd.flags & MMC_RSP_SPI_BUSY)
642	return mmc_spi_err_check(card);
643	return err;
644	}
645
646	/*
647	* Ensure RPMB, writes and R1B responses are completed by polling with
648	* CMD13. Note that, usually we don't need to poll when using HW busy
649	* detection, but here it's needed since some commands may indicate the
650	* error through the R1 status bits.
651	*/
652	if (idata->rpmb || idata->ic.write_flag || r1b_resp) {
653	struct mmc_blk_busy_data cb_data = {
654	.card = card,
655	};
656
657	err = __mmc_poll_for_busy(host: card->host, period_us: 0, timeout_ms: busy_timeout_ms,
658	busy_cb: &mmc_blk_busy_cb, cb_data: &cb_data);
659
660	idata->ic.response[0] = cb_data.status;
661	}
662
663	return err;
664	}
665
666	static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md,
667	struct mmc_ioc_cmd __user *ic_ptr,
668	struct mmc_rpmb_data *rpmb)
669	{
670	struct mmc_blk_ioc_data *idata;
671	struct mmc_blk_ioc_data *idatas[1];
672	struct mmc_queue *mq;
673	struct mmc_card *card;
674	int err = 0, ioc_err = 0;
675	struct request *req;
676
677	idata = mmc_blk_ioctl_copy_from_user(user: ic_ptr);
678	if (IS_ERR(ptr: idata))
679	return PTR_ERR(ptr: idata);
680	/* This will be NULL on non-RPMB ioctl():s */
681	idata->rpmb = rpmb;
682
683	card = md->queue.card;
684	if (IS_ERR(ptr: card)) {
685	err = PTR_ERR(ptr: card);
686	goto cmd_done;
687	}
688
689	/*
690	* Dispatch the ioctl() into the block request queue.
691	*/
692	mq = &md->queue;
693	req = blk_mq_alloc_request(q: mq->queue,
694	opf: idata->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, flags: 0);
695	if (IS_ERR(ptr: req)) {
696	err = PTR_ERR(ptr: req);
697	goto cmd_done;
698	}
699	idatas[0] = idata;
700	req_to_mmc_queue_req(rq: req)->drv_op =
701	rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
702	req_to_mmc_queue_req(rq: req)->drv_op_result = -EIO;
703	req_to_mmc_queue_req(rq: req)->drv_op_data = idatas;
704	req_to_mmc_queue_req(rq: req)->ioc_count = 1;
705	blk_execute_rq(rq: req, at_head: false);
706	ioc_err = req_to_mmc_queue_req(rq: req)->drv_op_result;
707	err = mmc_blk_ioctl_copy_to_user(ic_ptr, idata);
708	blk_mq_free_request(rq: req);
709
710	cmd_done:
711	kfree(objp: idata->buf);
712	kfree(objp: idata);
713	return ioc_err ? ioc_err : err;
714	}
715
716	static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md,
717	struct mmc_ioc_multi_cmd __user *user,
718	struct mmc_rpmb_data *rpmb)
719	{
720	struct mmc_blk_ioc_data **idata = NULL;
721	struct mmc_ioc_cmd __user *cmds = user->cmds;
722	struct mmc_card *card;
723	struct mmc_queue *mq;
724	int err = 0, ioc_err = 0;
725	__u64 num_of_cmds;
726	unsigned int i, n;
727	struct request *req;
728
729	if (copy_from_user(to: &num_of_cmds, from: &user->num_of_cmds,
730	n: sizeof(num_of_cmds)))
731	return -EFAULT;
732
733	if (!num_of_cmds)
734	return 0;
735
736	if (num_of_cmds > MMC_IOC_MAX_CMDS)
737	return -EINVAL;
738
739	n = num_of_cmds;
740	idata = kcalloc(n, sizeof(*idata), GFP_KERNEL);
741	if (!idata)
742	return -ENOMEM;
743
744	for (i = 0; i < n; i++) {
745	idata[i] = mmc_blk_ioctl_copy_from_user(user: &cmds[i]);
746	if (IS_ERR(ptr: idata[i])) {
747	err = PTR_ERR(ptr: idata[i]);
748	n = i;
749	goto cmd_err;
750	}
751	/* This will be NULL on non-RPMB ioctl():s */
752	idata[i]->rpmb = rpmb;
753	}
754
755	card = md->queue.card;
756	if (IS_ERR(ptr: card)) {
757	err = PTR_ERR(ptr: card);
758	goto cmd_err;
759	}
760
761
762	/*
763	* Dispatch the ioctl()s into the block request queue.
764	*/
765	mq = &md->queue;
766	req = blk_mq_alloc_request(q: mq->queue,
767	opf: idata[0]->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, flags: 0);
768	if (IS_ERR(ptr: req)) {
769	err = PTR_ERR(ptr: req);
770	goto cmd_err;
771	}
772	req_to_mmc_queue_req(rq: req)->drv_op =
773	rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
774	req_to_mmc_queue_req(rq: req)->drv_op_result = -EIO;
775	req_to_mmc_queue_req(rq: req)->drv_op_data = idata;
776	req_to_mmc_queue_req(rq: req)->ioc_count = n;
777	blk_execute_rq(rq: req, at_head: false);
778	ioc_err = req_to_mmc_queue_req(rq: req)->drv_op_result;
779
780	/* copy to user if data and response */
781	for (i = 0; i < n && !err; i++)
782	err = mmc_blk_ioctl_copy_to_user(ic_ptr: &cmds[i], idata: idata[i]);
783
784	blk_mq_free_request(rq: req);
785
786	cmd_err:
787	for (i = 0; i < n; i++) {
788	kfree(objp: idata[i]->buf);
789	kfree(objp: idata[i]);
790	}
791	kfree(objp: idata);
792	return ioc_err ? ioc_err : err;
793	}
794
795	static int mmc_blk_check_blkdev(struct block_device *bdev)
796	{
797	/*
798	* The caller must have CAP_SYS_RAWIO, and must be calling this on the
799	* whole block device, not on a partition. This prevents overspray
800	* between sibling partitions.
801	*/
802	if (!capable(CAP_SYS_RAWIO) || bdev_is_partition(bdev))
803	return -EPERM;
804	return 0;
805	}
806
807	static int mmc_blk_ioctl(struct block_device *bdev, blk_mode_t mode,
808	unsigned int cmd, unsigned long arg)
809	{
810	struct mmc_blk_data *md;
811	int ret;
812
813	switch (cmd) {
814	case MMC_IOC_CMD:
815	ret = mmc_blk_check_blkdev(bdev);
816	if (ret)
817	return ret;
818	md = mmc_blk_get(disk: bdev->bd_disk);
819	if (!md)
820	return -EINVAL;
821	ret = mmc_blk_ioctl_cmd(md,
822	ic_ptr: (struct mmc_ioc_cmd __user *)arg,
823	NULL);
824	mmc_blk_put(md);
825	return ret;
826	case MMC_IOC_MULTI_CMD:
827	ret = mmc_blk_check_blkdev(bdev);
828	if (ret)
829	return ret;
830	md = mmc_blk_get(disk: bdev->bd_disk);
831	if (!md)
832	return -EINVAL;
833	ret = mmc_blk_ioctl_multi_cmd(md,
834	user: (struct mmc_ioc_multi_cmd __user *)arg,
835	NULL);
836	mmc_blk_put(md);
837	return ret;
838	default:
839	return -EINVAL;
840	}
841	}
842
843	#ifdef CONFIG_COMPAT
844	static int mmc_blk_compat_ioctl(struct block_device *bdev, blk_mode_t mode,
845	unsigned int cmd, unsigned long arg)
846	{
847	return mmc_blk_ioctl(bdev, mode, cmd, arg: (unsigned long) compat_ptr(uptr: arg));
848	}
849	#endif
850
851	static int mmc_blk_alternative_gpt_sector(struct gendisk *disk,
852	sector_t *sector)
853	{
854	struct mmc_blk_data *md;
855	int ret;
856
857	md = mmc_blk_get(disk);
858	if (!md)
859	return -EINVAL;
860
861	if (md->queue.card)
862	ret = mmc_card_alternative_gpt_sector(card: md->queue.card, sector);
863	else
864	ret = -ENODEV;
865
866	mmc_blk_put(md);
867
868	return ret;
869	}
870
871	static const struct block_device_operations mmc_bdops = {
872	.open = mmc_blk_open,
873	.release = mmc_blk_release,
874	.getgeo = mmc_blk_getgeo,
875	.owner = THIS_MODULE,
876	.ioctl = mmc_blk_ioctl,
877	#ifdef CONFIG_COMPAT
878	.compat_ioctl = mmc_blk_compat_ioctl,
879	#endif
880	.alternative_gpt_sector = mmc_blk_alternative_gpt_sector,
881	};
882
883	static int mmc_blk_part_switch_pre(struct mmc_card *card,
884	unsigned int part_type)
885	{
886	const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_MASK;
887	const unsigned int rpmb = EXT_CSD_PART_CONFIG_ACC_RPMB;
888	int ret = 0;
889
890	if ((part_type & mask) == rpmb) {
891	if (card->ext_csd.cmdq_en) {
892	ret = mmc_cmdq_disable(card);
893	if (ret)
894	return ret;
895	}
896	mmc_retune_pause(host: card->host);
897	}
898
899	return ret;
900	}
901
902	static int mmc_blk_part_switch_post(struct mmc_card *card,
903	unsigned int part_type)
904	{
905	const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_MASK;
906	const unsigned int rpmb = EXT_CSD_PART_CONFIG_ACC_RPMB;
907	int ret = 0;
908
909	if ((part_type & mask) == rpmb) {
910	mmc_retune_unpause(host: card->host);
911	if (card->reenable_cmdq && !card->ext_csd.cmdq_en)
912	ret = mmc_cmdq_enable(card);
913	}
914
915	return ret;
916	}
917
918	static inline int mmc_blk_part_switch(struct mmc_card *card,
919	unsigned int part_type)
920	{
921	int ret = 0;
922	struct mmc_blk_data *main_md = dev_get_drvdata(dev: &card->dev);
923
924	if (main_md->part_curr == part_type)
925	return 0;
926
927	if (mmc_card_mmc(card)) {
928	u8 part_config = card->ext_csd.part_config;
929
930	ret = mmc_blk_part_switch_pre(card, part_type);
931	if (ret)
932	return ret;
933
934	part_config &= ~EXT_CSD_PART_CONFIG_ACC_MASK;
935	part_config |= part_type;
936
937	ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
938	EXT_CSD_PART_CONFIG, value: part_config,
939	timeout_ms: card->ext_csd.part_time);
940	if (ret) {
941	mmc_blk_part_switch_post(card, part_type);
942	return ret;
943	}
944
945	card->ext_csd.part_config = part_config;
946
947	ret = mmc_blk_part_switch_post(card, part_type: main_md->part_curr);
948	}
949
950	main_md->part_curr = part_type;
951	return ret;
952	}
953
954	static int mmc_sd_num_wr_blocks(struct mmc_card *card, u32 *written_blocks)
955	{
956	int err;
957	u32 result;
958	__be32 *blocks;
959	u8 resp_sz = mmc_card_ult_capacity(card) ? 8 : 4;
960
961	struct mmc_request mrq = {};
962	struct mmc_command cmd = {};
963	struct mmc_data data = {};
964	struct scatterlist sg;
965
966	err = mmc_app_cmd(host: card->host, card);
967	if (err)
968	return err;
969
970	cmd.opcode = SD_APP_SEND_NUM_WR_BLKS;
971	cmd.arg = 0;
972	cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC;
973
974	data.blksz = resp_sz;
975	data.blocks = 1;
976	data.flags = MMC_DATA_READ;
977	data.sg = &sg;
978	data.sg_len = 1;
979	mmc_set_data_timeout(data: &data, card);
980
981	mrq.cmd = &cmd;
982	mrq.data = &data;
983
984	blocks = kmalloc(resp_sz, GFP_NOIO);
985	if (!blocks)
986	return -ENOMEM;
987
988	sg_init_one(&sg, blocks, resp_sz);
989
990	mmc_wait_for_req(host: card->host, mrq: &mrq);
991
992	if (mmc_card_ult_capacity(card)) {
993	/*
994	* Normally, ACMD22 returns the number of written sectors as
995	* u32. SDUC, however, returns it as u64. This is not a
996	* superfluous requirement, because SDUC writes may exceed 2TB.
997	* For Linux mmc however, the previously write operation could
998	* not be more than the block layer limits, thus just make room
999	* for a u64 and cast the response back to u32.
1000	*/
1001	result = clamp_val(get_unaligned_be64(blocks), 0, UINT_MAX);
1002	} else {
1003	result = ntohl(*blocks);
1004	}
1005	kfree(objp: blocks);
1006
1007	if (cmd.error || data.error)
1008	return -EIO;
1009
1010	*written_blocks = result;
1011
1012	return 0;
1013	}
1014
1015	static unsigned int mmc_blk_clock_khz(struct mmc_host *host)
1016	{
1017	if (host->actual_clock)
1018	return host->actual_clock / 1000;
1019
1020	/* Clock may be subject to a divisor, fudge it by a factor of 2. */
1021	if (host->ios.clock)
1022	return host->ios.clock / 2000;
1023
1024	/* How can there be no clock */
1025	WARN_ON_ONCE(1);
1026	return 100; /* 100 kHz is minimum possible value */
1027	}
1028
1029	static unsigned int mmc_blk_data_timeout_ms(struct mmc_host *host,
1030	struct mmc_data *data)
1031	{
1032	unsigned int ms = DIV_ROUND_UP(data->timeout_ns, 1000000);
1033	unsigned int khz;
1034
1035	if (data->timeout_clks) {
1036	khz = mmc_blk_clock_khz(host);
1037	ms += DIV_ROUND_UP(data->timeout_clks, khz);
1038	}
1039
1040	return ms;
1041	}
1042
1043	/*
1044	* Attempts to reset the card and get back to the requested partition.
1045	* Therefore any error here must result in cancelling the block layer
1046	* request, it must not be reattempted without going through the mmc_blk
1047	* partition sanity checks.
1048	*/
1049	static int mmc_blk_reset(struct mmc_blk_data *md, struct mmc_host *host,
1050	int type)
1051	{
1052	int err;
1053	struct mmc_blk_data *main_md = dev_get_drvdata(dev: &host->card->dev);
1054
1055	if (md->reset_done & type)
1056	return -EEXIST;
1057
1058	md->reset_done |= type;
1059	err = mmc_hw_reset(card: host->card);
1060	/*
1061	* A successful reset will leave the card in the main partition, but
1062	* upon failure it might not be, so set it to MMC_BLK_PART_INVALID
1063	* in that case.
1064	*/
1065	main_md->part_curr = err ? MMC_BLK_PART_INVALID : main_md->part_type;
1066	if (err)
1067	return err;
1068	/* Ensure we switch back to the correct partition */
1069	if (mmc_blk_part_switch(card: host->card, part_type: md->part_type))
1070	/*
1071	* We have failed to get back into the correct
1072	* partition, so we need to abort the whole request.
1073	*/
1074	return -ENODEV;
1075	return 0;
1076	}
1077
1078	static inline void mmc_blk_reset_success(struct mmc_blk_data *md, int type)
1079	{
1080	md->reset_done &= ~type;
1081	}
1082
1083	static void mmc_blk_check_sbc(struct mmc_queue_req *mq_rq)
1084	{
1085	struct mmc_blk_ioc_data **idata = mq_rq->drv_op_data;
1086	int i;
1087
1088	for (i = 1; i < mq_rq->ioc_count; i++) {
1089	if (idata[i - 1]->ic.opcode == MMC_SET_BLOCK_COUNT &&
1090	mmc_op_multi(opcode: idata[i]->ic.opcode)) {
1091	idata[i - 1]->flags |= MMC_BLK_IOC_DROP;
1092	idata[i]->flags |= MMC_BLK_IOC_SBC;
1093	}
1094	}
1095	}
1096
1097	/*
1098	* The non-block commands come back from the block layer after it queued it and
1099	* processed it with all other requests and then they get issued in this
1100	* function.
1101	*/
1102	static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req)
1103	{
1104	struct mmc_queue_req *mq_rq;
1105	struct mmc_card *card = mq->card;
1106	struct mmc_blk_data *md = mq->blkdata;
1107	struct mmc_blk_ioc_data **idata;
1108	bool rpmb_ioctl;
1109	u8 **ext_csd;
1110	u32 status;
1111	int ret;
1112	int i;
1113
1114	mq_rq = req_to_mmc_queue_req(rq: req);
1115	rpmb_ioctl = (mq_rq->drv_op == MMC_DRV_OP_IOCTL_RPMB);
1116
1117	switch (mq_rq->drv_op) {
1118	case MMC_DRV_OP_IOCTL:
1119	if (card->ext_csd.cmdq_en) {
1120	ret = mmc_cmdq_disable(card);
1121	if (ret)
1122	break;
1123	}
1124
1125	mmc_blk_check_sbc(mq_rq);
1126
1127	fallthrough;
1128	case MMC_DRV_OP_IOCTL_RPMB:
1129	idata = mq_rq->drv_op_data;
1130	for (i = 0, ret = 0; i < mq_rq->ioc_count; i++) {
1131	ret = __mmc_blk_ioctl_cmd(card, md, idatas: idata, i);
1132	if (ret)
1133	break;
1134	}
1135	/* Always switch back to main area after RPMB access */
1136	if (rpmb_ioctl)
1137	mmc_blk_part_switch(card, part_type: 0);
1138	else if (card->reenable_cmdq && !card->ext_csd.cmdq_en)
1139	mmc_cmdq_enable(card);
1140	break;
1141	case MMC_DRV_OP_BOOT_WP:
1142	ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_BOOT_WP,
1143	value: card->ext_csd.boot_ro_lock |
1144	EXT_CSD_BOOT_WP_B_PWR_WP_EN,
1145	timeout_ms: card->ext_csd.part_time);
1146	if (ret)
1147	pr_err("%s: Locking boot partition ro until next power on failed: %d\n",
1148	md->disk->disk_name, ret);
1149	else
1150	card->ext_csd.boot_ro_lock |=
1151	EXT_CSD_BOOT_WP_B_PWR_WP_EN;
1152	break;
1153	case MMC_DRV_OP_GET_CARD_STATUS:
1154	ret = mmc_send_status(card, status: &status);
1155	if (!ret)
1156	ret = status;
1157	break;
1158	case MMC_DRV_OP_GET_EXT_CSD:
1159	ext_csd = mq_rq->drv_op_data;
1160	ret = mmc_get_ext_csd(card, new_ext_csd: ext_csd);
1161	break;
1162	default:
1163	pr_err("%s: unknown driver specific operation\n",
1164	md->disk->disk_name);
1165	ret = -EINVAL;
1166	break;
1167	}
1168	mq_rq->drv_op_result = ret;
1169	blk_mq_end_request(rq: req, error: ret ? BLK_STS_IOERR : BLK_STS_OK);
1170	}
1171
1172	static void mmc_blk_issue_erase_rq(struct mmc_queue *mq, struct request *req,
1173	int type, unsigned int erase_arg)
1174	{
1175	struct mmc_blk_data *md = mq->blkdata;
1176	struct mmc_card *card = md->queue.card;
1177	unsigned int nr;
1178	sector_t from;
1179	int err = 0;
1180	blk_status_t status = BLK_STS_OK;
1181
1182	if (!mmc_card_can_erase(card)) {
1183	status = BLK_STS_NOTSUPP;
1184	goto fail;
1185	}
1186
1187	from = blk_rq_pos(rq: req);
1188	nr = blk_rq_sectors(rq: req);
1189
1190	do {
1191	err = 0;
1192	if (card->quirks & MMC_QUIRK_INAND_CMD38) {
1193	err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
1194	INAND_CMD38_ARG_EXT_CSD,
1195	value: erase_arg == MMC_TRIM_ARG ?
1196	INAND_CMD38_ARG_TRIM :
1197	INAND_CMD38_ARG_ERASE,
1198	timeout_ms: card->ext_csd.generic_cmd6_time);
1199	}
1200	if (!err)
1201	err = mmc_erase(card, from, nr, arg: erase_arg);
1202	} while (err == -EIO && !mmc_blk_reset(md, host: card->host, type));
1203	if (err)
1204	status = BLK_STS_IOERR;
1205	else
1206	mmc_blk_reset_success(md, type);
1207	fail:
1208	blk_mq_end_request(rq: req, error: status);
1209	}
1210
1211	static void mmc_blk_issue_trim_rq(struct mmc_queue *mq, struct request *req)
1212	{
1213	mmc_blk_issue_erase_rq(mq, req, MMC_BLK_TRIM, MMC_TRIM_ARG);
1214	}
1215
1216	static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)
1217	{
1218	struct mmc_blk_data *md = mq->blkdata;
1219	struct mmc_card *card = md->queue.card;
1220	unsigned int arg = card->erase_arg;
1221
1222	if (mmc_card_broken_sd_discard(c: card))
1223	arg = SD_ERASE_ARG;
1224
1225	mmc_blk_issue_erase_rq(mq, req, MMC_BLK_DISCARD, erase_arg: arg);
1226	}
1227
1228	static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
1229	struct request *req)
1230	{
1231	struct mmc_blk_data *md = mq->blkdata;
1232	struct mmc_card *card = md->queue.card;
1233	unsigned int nr, arg;
1234	sector_t from;
1235	int err = 0, type = MMC_BLK_SECDISCARD;
1236	blk_status_t status = BLK_STS_OK;
1237
1238	if (!(mmc_card_can_secure_erase_trim(card))) {
1239	status = BLK_STS_NOTSUPP;
1240	goto out;
1241	}
1242
1243	from = blk_rq_pos(rq: req);
1244	nr = blk_rq_sectors(rq: req);
1245
1246	if (mmc_card_can_trim(card) && !mmc_erase_group_aligned(card, from, nr))
1247	arg = MMC_SECURE_TRIM1_ARG;
1248	else
1249	arg = MMC_SECURE_ERASE_ARG;
1250
1251	retry:
1252	if (card->quirks & MMC_QUIRK_INAND_CMD38) {
1253	err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
1254	INAND_CMD38_ARG_EXT_CSD,
1255	value: arg == MMC_SECURE_TRIM1_ARG ?
1256	INAND_CMD38_ARG_SECTRIM1 :
1257	INAND_CMD38_ARG_SECERASE,
1258	timeout_ms: card->ext_csd.generic_cmd6_time);
1259	if (err)
1260	goto out_retry;
1261	}
1262
1263	err = mmc_erase(card, from, nr, arg);
1264	if (err == -EIO)
1265	goto out_retry;
1266	if (err) {
1267	status = BLK_STS_IOERR;
1268	goto out;
1269	}
1270
1271	if (arg == MMC_SECURE_TRIM1_ARG) {
1272	if (card->quirks & MMC_QUIRK_INAND_CMD38) {
1273	err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
1274	INAND_CMD38_ARG_EXT_CSD,
1275	INAND_CMD38_ARG_SECTRIM2,
1276	timeout_ms: card->ext_csd.generic_cmd6_time);
1277	if (err)
1278	goto out_retry;
1279	}
1280
1281	err = mmc_erase(card, from, nr, MMC_SECURE_TRIM2_ARG);
1282	if (err == -EIO)
1283	goto out_retry;
1284	if (err) {
1285	status = BLK_STS_IOERR;
1286	goto out;
1287	}
1288	}
1289
1290	out_retry:
1291	if (err && !mmc_blk_reset(md, host: card->host, type))
1292	goto retry;
1293	if (!err)
1294	mmc_blk_reset_success(md, type);
1295	out:
1296	blk_mq_end_request(rq: req, error: status);
1297	}
1298
1299	static void mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)
1300	{
1301	struct mmc_blk_data *md = mq->blkdata;
1302	struct mmc_card *card = md->queue.card;
1303	int ret = 0;
1304
1305	ret = mmc_flush_cache(host: card->host);
1306	blk_mq_end_request(rq: req, error: ret ? BLK_STS_IOERR : BLK_STS_OK);
1307	}
1308
1309	/*
1310	* Reformat current write as a reliable write, supporting
1311	* both legacy and the enhanced reliable write MMC cards.
1312	* In each transfer we'll handle only as much as a single
1313	* reliable write can handle, thus finish the request in
1314	* partial completions.
1315	*/
1316	static inline void mmc_apply_rel_rw(struct mmc_blk_request *brq,
1317	struct mmc_card *card,
1318	struct request *req)
1319	{
1320	if (!(card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN)) {
1321	/* Legacy mode imposes restrictions on transfers. */
1322	if (!IS_ALIGNED(blk_rq_pos(req), card->ext_csd.rel_sectors))
1323	brq->data.blocks = 1;
1324
1325	if (brq->data.blocks > card->ext_csd.rel_sectors)
1326	brq->data.blocks = card->ext_csd.rel_sectors;
1327	else if (brq->data.blocks < card->ext_csd.rel_sectors)
1328	brq->data.blocks = 1;
1329	}
1330	}
1331
1332	#define CMD_ERRORS_EXCL_OOR \
1333	(R1_ADDRESS_ERROR | /* Misaligned address */ \
1334	R1_BLOCK_LEN_ERROR | /* Transferred block length incorrect */\
1335	R1_WP_VIOLATION | /* Tried to write to protected block */ \
1336	R1_CARD_ECC_FAILED | /* Card ECC failed */ \
1337	R1_CC_ERROR | /* Card controller error */ \
1338	R1_ERROR) /* General/unknown error */
1339
1340	#define CMD_ERRORS \
1341	(CMD_ERRORS_EXCL_OOR | \
1342	R1_OUT_OF_RANGE) /* Command argument out of range */ \
1343
1344	static void mmc_blk_eval_resp_error(struct mmc_blk_request *brq)
1345	{
1346	u32 val;
1347
1348	/*
1349	* Per the SD specification(physical layer version 4.10)[1],
1350	* section 4.3.3, it explicitly states that "When the last
1351	* block of user area is read using CMD18, the host should
1352	* ignore OUT_OF_RANGE error that may occur even the sequence
1353	* is correct". And JESD84-B51 for eMMC also has a similar
1354	* statement on section 6.8.3.
1355	*
1356	* Multiple block read/write could be done by either predefined
1357	* method, namely CMD23, or open-ending mode. For open-ending mode,
1358	* we should ignore the OUT_OF_RANGE error as it's normal behaviour.
1359	*
1360	* However the spec[1] doesn't tell us whether we should also
1361	* ignore that for predefined method. But per the spec[1], section
1362	* 4.15 Set Block Count Command, it says"If illegal block count
1363	* is set, out of range error will be indicated during read/write
1364	* operation (For example, data transfer is stopped at user area
1365	* boundary)." In another word, we could expect a out of range error
1366	* in the response for the following CMD18/25. And if argument of
1367	* CMD23 + the argument of CMD18/25 exceed the max number of blocks,
1368	* we could also expect to get a -ETIMEDOUT or any error number from
1369	* the host drivers due to missing data response(for write)/data(for
1370	* read), as the cards will stop the data transfer by itself per the
1371	* spec. So we only need to check R1_OUT_OF_RANGE for open-ending mode.
1372	*/
1373
1374	if (!brq->stop.error) {
1375	bool oor_with_open_end;
1376	/* If there is no error yet, check R1 response */
1377
1378	val = brq->stop.resp[0] & CMD_ERRORS;
1379	oor_with_open_end = val & R1_OUT_OF_RANGE && !brq->mrq.sbc;
1380
1381	if (val && !oor_with_open_end)
1382	brq->stop.error = -EIO;
1383	}
1384	}
1385
1386	static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq,
1387	int recovery_mode, bool *do_rel_wr_p,
1388	bool *do_data_tag_p)
1389	{
1390	struct mmc_blk_data *md = mq->blkdata;
1391	struct mmc_card *card = md->queue.card;
1392	struct mmc_blk_request *brq = &mqrq->brq;
1393	struct request *req = mmc_queue_req_to_req(mqr: mqrq);
1394	bool do_rel_wr, do_data_tag;
1395
1396	/*
1397	* Reliable writes are used to implement Forced Unit Access and
1398	* are supported only on MMCs.
1399	*/
1400	do_rel_wr = (req->cmd_flags & REQ_FUA) &&
1401	rq_data_dir(req) == WRITE &&
1402	(md->flags & MMC_BLK_REL_WR);
1403
1404	memset(brq, 0, sizeof(struct mmc_blk_request));
1405
1406	mmc_crypto_prepare_req(mqrq);
1407
1408	brq->mrq.data = &brq->data;
1409	brq->mrq.tag = req->tag;
1410
1411	brq->stop.opcode = MMC_STOP_TRANSMISSION;
1412	brq->stop.arg = 0;
1413
1414	if (rq_data_dir(req) == READ) {
1415	brq->data.flags = MMC_DATA_READ;
1416	brq->stop.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC;
1417	} else {
1418	brq->data.flags = MMC_DATA_WRITE;
1419	brq->stop.flags = MMC_RSP_SPI_R1B | MMC_RSP_R1B | MMC_CMD_AC;
1420	}
1421
1422	brq->data.blksz = 512;
1423	brq->data.blocks = blk_rq_sectors(rq: req);
1424	brq->data.blk_addr = blk_rq_pos(rq: req);
1425
1426	/*
1427	* The command queue supports 2 priorities: "high" (1) and "simple" (0).
1428	* The eMMC will give "high" priority tasks priority over "simple"
1429	* priority tasks. Here we always set "simple" priority by not setting
1430	* MMC_DATA_PRIO.
1431	*/
1432
1433	/*
1434	* The block layer doesn't support all sector count
1435	* restrictions, so we need to be prepared for too big
1436	* requests.
1437	*/
1438	if (brq->data.blocks > card->host->max_blk_count)
1439	brq->data.blocks = card->host->max_blk_count;
1440
1441	if (brq->data.blocks > 1) {
1442	/*
1443	* Some SD cards in SPI mode return a CRC error or even lock up
1444	* completely when trying to read the last block using a
1445	* multiblock read command.
1446	*/
1447	if (mmc_host_is_spi(card->host) && (rq_data_dir(req) == READ) &&
1448	(blk_rq_pos(rq: req) + blk_rq_sectors(rq: req) ==
1449	get_capacity(disk: md->disk)))
1450	brq->data.blocks--;
1451
1452	/*
1453	* After a read error, we redo the request one (native) sector
1454	* at a time in order to accurately determine which
1455	* sectors can be read successfully.
1456	*/
1457	if (recovery_mode)
1458	brq->data.blocks = queue_physical_block_size(q: mq->queue) >> 9;
1459
1460	/*
1461	* Some controllers have HW issues while operating
1462	* in multiple I/O mode
1463	*/
1464	if (card->host->ops->multi_io_quirk)
1465	brq->data.blocks = card->host->ops->multi_io_quirk(card,
1466	(rq_data_dir(req) == READ) ?
1467	MMC_DATA_READ : MMC_DATA_WRITE,
1468	brq->data.blocks);
1469	}
1470
1471	if (do_rel_wr) {
1472	mmc_apply_rel_rw(brq, card, req);
1473	brq->data.flags |= MMC_DATA_REL_WR;
1474	}
1475
1476	/*
1477	* Data tag is used only during writing meta data to speed
1478	* up write and any subsequent read of this meta data
1479	*/
1480	do_data_tag = card->ext_csd.data_tag_unit_size &&
1481	(req->cmd_flags & REQ_META) &&
1482	(rq_data_dir(req) == WRITE) &&
1483	((brq->data.blocks * brq->data.blksz) >=
1484	card->ext_csd.data_tag_unit_size);
1485
1486	if (do_data_tag)
1487	brq->data.flags |= MMC_DATA_DAT_TAG;
1488
1489	mmc_set_data_timeout(data: &brq->data, card);
1490
1491	brq->data.sg = mqrq->sg;
1492	brq->data.sg_len = mmc_queue_map_sg(mq, mqrq);
1493
1494	/*
1495	* Adjust the sg list so it is the same size as the
1496	* request.
1497	*/
1498	if (brq->data.blocks != blk_rq_sectors(rq: req)) {
1499	int i, data_size = brq->data.blocks << 9;
1500	struct scatterlist *sg;
1501
1502	for_each_sg(brq->data.sg, sg, brq->data.sg_len, i) {
1503	data_size -= sg->length;
1504	if (data_size <= 0) {
1505	sg->length += data_size;
1506	i++;
1507	break;
1508	}
1509	}
1510	brq->data.sg_len = i;
1511	}
1512
1513	if (do_rel_wr_p)
1514	*do_rel_wr_p = do_rel_wr;
1515
1516	if (do_data_tag_p)
1517	*do_data_tag_p = do_data_tag;
1518	}
1519
1520	#define MMC_CQE_RETRIES 2
1521
1522	static void mmc_blk_cqe_complete_rq(struct mmc_queue *mq, struct request *req)
1523	{
1524	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
1525	struct mmc_request *mrq = &mqrq->brq.mrq;
1526	struct request_queue *q = req->q;
1527	struct mmc_host *host = mq->card->host;
1528	enum mmc_issue_type issue_type = mmc_issue_type(mq, req);
1529	unsigned long flags;
1530	bool put_card;
1531	int err;
1532
1533	mmc_cqe_post_req(host, mrq);
1534
1535	if (mrq->cmd && mrq->cmd->error)
1536	err = mrq->cmd->error;
1537	else if (mrq->data && mrq->data->error)
1538	err = mrq->data->error;
1539	else
1540	err = 0;
1541
1542	if (err) {
1543	if (mqrq->retries++ < MMC_CQE_RETRIES)
1544	blk_mq_requeue_request(rq: req, kick_requeue_list: true);
1545	else
1546	blk_mq_end_request(rq: req, BLK_STS_IOERR);
1547	} else if (mrq->data) {
1548	if (blk_update_request(rq: req, BLK_STS_OK, nr_bytes: mrq->data->bytes_xfered))
1549	blk_mq_requeue_request(rq: req, kick_requeue_list: true);
1550	else
1551	__blk_mq_end_request(rq: req, BLK_STS_OK);
1552	} else if (mq->in_recovery) {
1553	blk_mq_requeue_request(rq: req, kick_requeue_list: true);
1554	} else {
1555	blk_mq_end_request(rq: req, BLK_STS_OK);
1556	}
1557
1558	spin_lock_irqsave(&mq->lock, flags);
1559
1560	mq->in_flight[issue_type] -= 1;
1561
1562	put_card = (mmc_tot_in_flight(mq) == 0);
1563
1564	mmc_cqe_check_busy(mq);
1565
1566	spin_unlock_irqrestore(lock: &mq->lock, flags);
1567
1568	if (!mq->cqe_busy)
1569	blk_mq_run_hw_queues(q, async: true);
1570
1571	if (put_card)
1572	mmc_put_card(card: mq->card, ctx: &mq->ctx);
1573	}
1574
1575	void mmc_blk_cqe_recovery(struct mmc_queue *mq)
1576	{
1577	struct mmc_card *card = mq->card;
1578	struct mmc_host *host = card->host;
1579	int err;
1580
1581	pr_debug("%s: CQE recovery start\n", mmc_hostname(host));
1582
1583	err = mmc_cqe_recovery(host);
1584	if (err)
1585	mmc_blk_reset(md: mq->blkdata, host, MMC_BLK_CQE_RECOVERY);
1586	mmc_blk_reset_success(md: mq->blkdata, MMC_BLK_CQE_RECOVERY);
1587
1588	pr_debug("%s: CQE recovery done\n", mmc_hostname(host));
1589	}
1590
1591	static void mmc_blk_cqe_req_done(struct mmc_request *mrq)
1592	{
1593	struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
1594	brq.mrq);
1595	struct request *req = mmc_queue_req_to_req(mqr: mqrq);
1596	struct request_queue *q = req->q;
1597	struct mmc_queue *mq = q->queuedata;
1598
1599	/*
1600	* Block layer timeouts race with completions which means the normal
1601	* completion path cannot be used during recovery.
1602	*/
1603	if (mq->in_recovery)
1604	mmc_blk_cqe_complete_rq(mq, req);
1605	else if (likely(!blk_should_fake_timeout(req->q)))
1606	blk_mq_complete_request(rq: req);
1607	}
1608
1609	static int mmc_blk_cqe_start_req(struct mmc_host *host, struct mmc_request *mrq)
1610	{
1611	mrq->done = mmc_blk_cqe_req_done;
1612	mrq->recovery_notifier = mmc_cqe_recovery_notifier;
1613
1614	return mmc_cqe_start_req(host, mrq);
1615	}
1616
1617	static struct mmc_request *mmc_blk_cqe_prep_dcmd(struct mmc_queue_req *mqrq,
1618	struct request *req)
1619	{
1620	struct mmc_blk_request *brq = &mqrq->brq;
1621
1622	memset(brq, 0, sizeof(*brq));
1623
1624	brq->mrq.cmd = &brq->cmd;
1625	brq->mrq.tag = req->tag;
1626
1627	return &brq->mrq;
1628	}
1629
1630	static int mmc_blk_cqe_issue_flush(struct mmc_queue *mq, struct request *req)
1631	{
1632	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
1633	struct mmc_request *mrq = mmc_blk_cqe_prep_dcmd(mqrq, req);
1634
1635	mrq->cmd->opcode = MMC_SWITCH;
1636	mrq->cmd->arg = (MMC_SWITCH_MODE_WRITE_BYTE << 24) |
1637	(EXT_CSD_FLUSH_CACHE << 16) |
1638	(1 << 8) |
1639	EXT_CSD_CMD_SET_NORMAL;
1640	mrq->cmd->flags = MMC_CMD_AC | MMC_RSP_R1B;
1641
1642	return mmc_blk_cqe_start_req(host: mq->card->host, mrq);
1643	}
1644
1645	static int mmc_blk_hsq_issue_rw_rq(struct mmc_queue *mq, struct request *req)
1646	{
1647	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
1648	struct mmc_host *host = mq->card->host;
1649	int err;
1650
1651	mmc_blk_rw_rq_prep(mqrq, card: mq->card, recovery_mode: 0, mq);
1652	mqrq->brq.mrq.done = mmc_blk_hsq_req_done;
1653	mmc_pre_req(host, mrq: &mqrq->brq.mrq);
1654
1655	err = mmc_cqe_start_req(host, mrq: &mqrq->brq.mrq);
1656	if (err)
1657	mmc_post_req(host, mrq: &mqrq->brq.mrq, err);
1658
1659	return err;
1660	}
1661
1662	static int mmc_blk_cqe_issue_rw_rq(struct mmc_queue *mq, struct request *req)
1663	{
1664	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
1665	struct mmc_host *host = mq->card->host;
1666
1667	if (host->hsq_enabled)
1668	return mmc_blk_hsq_issue_rw_rq(mq, req);
1669
1670	mmc_blk_data_prep(mq, mqrq, recovery_mode: 0, NULL, NULL);
1671
1672	return mmc_blk_cqe_start_req(host: mq->card->host, mrq: &mqrq->brq.mrq);
1673	}
1674
1675	static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
1676	struct mmc_card *card,
1677	int recovery_mode,
1678	struct mmc_queue *mq)
1679	{
1680	u32 readcmd, writecmd;
1681	struct mmc_blk_request *brq = &mqrq->brq;
1682	struct request *req = mmc_queue_req_to_req(mqr: mqrq);
1683	struct mmc_blk_data *md = mq->blkdata;
1684	bool do_rel_wr, do_data_tag;
1685
1686	mmc_blk_data_prep(mq, mqrq, recovery_mode, do_rel_wr_p: &do_rel_wr, do_data_tag_p: &do_data_tag);
1687
1688	brq->mrq.cmd = &brq->cmd;
1689
1690	brq->cmd.arg = blk_rq_pos(rq: req);
1691	if (!mmc_card_blockaddr(card))
1692	brq->cmd.arg <<= 9;
1693	brq->cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC;
1694
1695	if (brq->data.blocks > 1 || do_rel_wr) {
1696	/* SPI multiblock writes terminate using a special
1697	* token, not a STOP_TRANSMISSION request.
1698	*/
1699	if (!mmc_host_is_spi(card->host) ||
1700	rq_data_dir(req) == READ)
1701	brq->mrq.stop = &brq->stop;
1702	readcmd = MMC_READ_MULTIPLE_BLOCK;
1703	writecmd = MMC_WRITE_MULTIPLE_BLOCK;
1704	} else {
1705	brq->mrq.stop = NULL;
1706	readcmd = MMC_READ_SINGLE_BLOCK;
1707	writecmd = MMC_WRITE_BLOCK;
1708	}
1709	brq->cmd.opcode = rq_data_dir(req) == READ ? readcmd : writecmd;
1710
1711	/*
1712	* Pre-defined multi-block transfers are preferable to
1713	* open ended-ones (and necessary for reliable writes).
1714	* However, it is not sufficient to just send CMD23,
1715	* and avoid the final CMD12, as on an error condition
1716	* CMD12 (stop) needs to be sent anyway. This, coupled
1717	* with Auto-CMD23 enhancements provided by some
1718	* hosts, means that the complexity of dealing
1719	* with this is best left to the host. If CMD23 is
1720	* supported by card and host, we'll fill sbc in and let
1721	* the host deal with handling it correctly. This means
1722	* that for hosts that don't expose MMC_CAP_CMD23, no
1723	* change of behavior will be observed.
1724	*
1725	* N.B: Some MMC cards experience perf degradation.
1726	* We'll avoid using CMD23-bounded multiblock writes for
1727	* these, while retaining features like reliable writes.
1728	*/
1729	if ((md->flags & MMC_BLK_CMD23) && mmc_op_multi(opcode: brq->cmd.opcode) &&
1730	(do_rel_wr || !mmc_card_blk_no_cmd23(c: card) || do_data_tag)) {
1731	brq->sbc.opcode = MMC_SET_BLOCK_COUNT;
1732	brq->sbc.arg = brq->data.blocks |
1733	(do_rel_wr ? (1 << 31) : 0) |
1734	(do_data_tag ? (1 << 29) : 0);
1735	brq->sbc.flags = MMC_RSP_R1 | MMC_CMD_AC;
1736	brq->mrq.sbc = &brq->sbc;
1737	}
1738
1739	if (mmc_card_ult_capacity(card)) {
1740	brq->cmd.ext_addr = blk_rq_pos(rq: req) >> 32;
1741	brq->cmd.has_ext_addr = true;
1742	}
1743	}
1744
1745	#define MMC_MAX_RETRIES 5
1746	#define MMC_DATA_RETRIES 2
1747	#define MMC_NO_RETRIES (MMC_MAX_RETRIES + 1)
1748
1749	static int mmc_blk_send_stop(struct mmc_card *card, unsigned int timeout)
1750	{
1751	struct mmc_command cmd = {
1752	.opcode = MMC_STOP_TRANSMISSION,
1753	.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC,
1754	/* Some hosts wait for busy anyway, so provide a busy timeout */
1755	.busy_timeout = timeout,
1756	};
1757
1758	return mmc_wait_for_cmd(host: card->host, cmd: &cmd, retries: 5);
1759	}
1760
1761	static int mmc_blk_fix_state(struct mmc_card *card, struct request *req)
1762	{
1763	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
1764	struct mmc_blk_request *brq = &mqrq->brq;
1765	unsigned int timeout = mmc_blk_data_timeout_ms(host: card->host, data: &brq->data);
1766	int err;
1767
1768	mmc_retune_hold_now(host: card->host);
1769
1770	mmc_blk_send_stop(card, timeout);
1771
1772	err = mmc_poll_for_busy(card, timeout_ms: timeout, retry_crc_err: false, busy_cmd: MMC_BUSY_IO);
1773
1774	mmc_retune_release(host: card->host);
1775
1776	return err;
1777	}
1778
1779	#define MMC_READ_SINGLE_RETRIES 2
1780
1781	/* Single (native) sector read during recovery */
1782	static void mmc_blk_read_single(struct mmc_queue *mq, struct request *req)
1783	{
1784	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
1785	struct mmc_request *mrq = &mqrq->brq.mrq;
1786	struct mmc_card *card = mq->card;
1787	struct mmc_host *host = card->host;
1788	blk_status_t error = BLK_STS_OK;
1789	size_t bytes_per_read = queue_physical_block_size(q: mq->queue);
1790
1791	do {
1792	u32 status;
1793	int err;
1794	int retries = 0;
1795
1796	while (retries++ <= MMC_READ_SINGLE_RETRIES) {
1797	mmc_blk_rw_rq_prep(mqrq, card, recovery_mode: 1, mq);
1798
1799	mmc_wait_for_req(host, mrq);
1800
1801	err = mmc_send_status(card, status: &status);
1802	if (err)
1803	goto error_exit;
1804
1805	if (!mmc_host_is_spi(host) &&
1806	!mmc_ready_for_data(status)) {
1807	err = mmc_blk_fix_state(card, req);
1808	if (err)
1809	goto error_exit;
1810	}
1811
1812	if (!mrq->cmd->error)
1813	break;
1814	}
1815
1816	if (mrq->cmd->error ||
1817	mrq->data->error ||
1818	(!mmc_host_is_spi(host) &&
1819	(mrq->cmd->resp[0] & CMD_ERRORS || status & CMD_ERRORS)))
1820	error = BLK_STS_IOERR;
1821	else
1822	error = BLK_STS_OK;
1823
1824	} while (blk_update_request(rq: req, error, nr_bytes: bytes_per_read));
1825
1826	return;
1827
1828	error_exit:
1829	mrq->data->bytes_xfered = 0;
1830	blk_update_request(rq: req, BLK_STS_IOERR, nr_bytes: bytes_per_read);
1831	/* Let it try the remaining request again */
1832	if (mqrq->retries > MMC_MAX_RETRIES - 1)
1833	mqrq->retries = MMC_MAX_RETRIES - 1;
1834	}
1835
1836	static inline bool mmc_blk_oor_valid(struct mmc_blk_request *brq)
1837	{
1838	return !!brq->mrq.sbc;
1839	}
1840
1841	static inline u32 mmc_blk_stop_err_bits(struct mmc_blk_request *brq)
1842	{
1843	return mmc_blk_oor_valid(brq) ? CMD_ERRORS : CMD_ERRORS_EXCL_OOR;
1844	}
1845
1846	/*
1847	* Check for errors the host controller driver might not have seen such as
1848	* response mode errors or invalid card state.
1849	*/
1850	static bool mmc_blk_status_error(struct request *req, u32 status)
1851	{
1852	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
1853	struct mmc_blk_request *brq = &mqrq->brq;
1854	struct mmc_queue *mq = req->q->queuedata;
1855	u32 stop_err_bits;
1856
1857	if (mmc_host_is_spi(mq->card->host))
1858	return false;
1859
1860	stop_err_bits = mmc_blk_stop_err_bits(brq);
1861
1862	return brq->cmd.resp[0] & CMD_ERRORS ||
1863	brq->stop.resp[0] & stop_err_bits ||
1864	status & stop_err_bits ||
1865	(rq_data_dir(req) == WRITE && !mmc_ready_for_data(status));
1866	}
1867
1868	static inline bool mmc_blk_cmd_started(struct mmc_blk_request *brq)
1869	{
1870	return !brq->sbc.error && !brq->cmd.error &&
1871	!(brq->cmd.resp[0] & CMD_ERRORS);
1872	}
1873
1874	/*
1875	* Requests are completed by mmc_blk_mq_complete_rq() which sets simple
1876	* policy:
1877	* 1. A request that has transferred at least some data is considered
1878	* successful and will be requeued if there is remaining data to
1879	* transfer.
1880	* 2. Otherwise the number of retries is incremented and the request
1881	* will be requeued if there are remaining retries.
1882	* 3. Otherwise the request will be errored out.
1883	* That means mmc_blk_mq_complete_rq() is controlled by bytes_xfered and
1884	* mqrq->retries. So there are only 4 possible actions here:
1885	* 1. do not accept the bytes_xfered value i.e. set it to zero
1886	* 2. change mqrq->retries to determine the number of retries
1887	* 3. try to reset the card
1888	* 4. read one sector at a time
1889	*/
1890	static void mmc_blk_mq_rw_recovery(struct mmc_queue *mq, struct request *req)
1891	{
1892	int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE;
1893	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
1894	struct mmc_blk_request *brq = &mqrq->brq;
1895	struct mmc_blk_data *md = mq->blkdata;
1896	struct mmc_card *card = mq->card;
1897	u32 status;
1898	u32 blocks;
1899	int err;
1900
1901	/*
1902	* Some errors the host driver might not have seen. Set the number of
1903	* bytes transferred to zero in that case.
1904	*/
1905	err = __mmc_send_status(card, status: &status, retries: 0);
1906	if (err || mmc_blk_status_error(req, status))
1907	brq->data.bytes_xfered = 0;
1908
1909	mmc_retune_release(host: card->host);
1910
1911	/*
1912	* Try again to get the status. This also provides an opportunity for
1913	* re-tuning.
1914	*/
1915	if (err)
1916	err = __mmc_send_status(card, status: &status, retries: 0);
1917
1918	/*
1919	* Nothing more to do after the number of bytes transferred has been
1920	* updated and there is no card.
1921	*/
1922	if (err && mmc_detect_card_removed(host: card->host))
1923	return;
1924
1925	/* Try to get back to "tran" state */
1926	if (!mmc_host_is_spi(mq->card->host) &&
1927	(err || !mmc_ready_for_data(status)))
1928	err = mmc_blk_fix_state(card: mq->card, req);
1929
1930	/*
1931	* Special case for SD cards where the card might record the number of
1932	* blocks written.
1933	*/
1934	if (!err && mmc_blk_cmd_started(brq) && mmc_card_sd(card) &&
1935	rq_data_dir(req) == WRITE) {
1936	if (mmc_sd_num_wr_blocks(card, written_blocks: &blocks))
1937	brq->data.bytes_xfered = 0;
1938	else
1939	brq->data.bytes_xfered = blocks << 9;
1940	}
1941
1942	/* Reset if the card is in a bad state */
1943	if (!mmc_host_is_spi(mq->card->host) &&
1944	err && mmc_blk_reset(md, host: card->host, type)) {
1945	pr_err("%s: recovery failed!\n", req->q->disk->disk_name);
1946	mqrq->retries = MMC_NO_RETRIES;
1947	return;
1948	}
1949
1950	/*
1951	* If anything was done, just return and if there is anything remaining
1952	* on the request it will get requeued.
1953	*/
1954	if (brq->data.bytes_xfered)
1955	return;
1956
1957	/* Reset before last retry */
1958	if (mqrq->retries + 1 == MMC_MAX_RETRIES &&
1959	mmc_blk_reset(md, host: card->host, type))
1960	return;
1961
1962	/* Command errors fail fast, so use all MMC_MAX_RETRIES */
1963	if (brq->sbc.error || brq->cmd.error)
1964	return;
1965
1966	/* Reduce the remaining retries for data errors */
1967	if (mqrq->retries < MMC_MAX_RETRIES - MMC_DATA_RETRIES) {
1968	mqrq->retries = MMC_MAX_RETRIES - MMC_DATA_RETRIES;
1969	return;
1970	}
1971
1972	if (rq_data_dir(req) == READ && brq->data.blocks >
1973	queue_physical_block_size(q: mq->queue) >> 9) {
1974	/* Read one (native) sector at a time */
1975	mmc_blk_read_single(mq, req);
1976	return;
1977	}
1978	}
1979
1980	static inline bool mmc_blk_rq_error(struct mmc_blk_request *brq)
1981	{
1982	mmc_blk_eval_resp_error(brq);
1983
1984	return brq->sbc.error || brq->cmd.error || brq->stop.error ||
1985	brq->data.error || brq->cmd.resp[0] & CMD_ERRORS;
1986	}
1987
1988	static int mmc_spi_err_check(struct mmc_card *card)
1989	{
1990	u32 status = 0;
1991	int err;
1992
1993	/*
1994	* SPI does not have a TRAN state we have to wait on, instead the
1995	* card is ready again when it no longer holds the line LOW.
1996	* We still have to ensure two things here before we know the write
1997	* was successful:
1998	* 1. The card has not disconnected during busy and we actually read our
1999	* own pull-up, thinking it was still connected, so ensure it
2000	* still responds.
2001	* 2. Check for any error bits, in particular R1_SPI_IDLE to catch a
2002	* just reconnected card after being disconnected during busy.
2003	*/
2004	err = __mmc_send_status(card, status: &status, retries: 0);
2005	if (err)
2006	return err;
2007	/* All R1 and R2 bits of SPI are errors in our case */
2008	if (status)
2009	return -EIO;
2010	return 0;
2011	}
2012
2013	static int mmc_blk_busy_cb(void *cb_data, bool *busy)
2014	{
2015	struct mmc_blk_busy_data *data = cb_data;
2016	u32 status = 0;
2017	int err;
2018
2019	err = mmc_send_status(card: data->card, status: &status);
2020	if (err)
2021	return err;
2022
2023	/* Accumulate response error bits. */
2024	data->status |= status;
2025
2026	*busy = !mmc_ready_for_data(status);
2027	return 0;
2028	}
2029
2030	static int mmc_blk_card_busy(struct mmc_card *card, struct request *req)
2031	{
2032	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
2033	struct mmc_blk_busy_data cb_data;
2034	int err;
2035
2036	if (rq_data_dir(req) == READ)
2037	return 0;
2038
2039	if (mmc_host_is_spi(card->host)) {
2040	err = mmc_spi_err_check(card);
2041	if (err)
2042	mqrq->brq.data.bytes_xfered = 0;
2043	return err;
2044	}
2045
2046	cb_data.card = card;
2047	cb_data.status = 0;
2048	err = __mmc_poll_for_busy(host: card->host, period_us: 0, MMC_BLK_TIMEOUT_MS,
2049	busy_cb: &mmc_blk_busy_cb, cb_data: &cb_data);
2050
2051	/*
2052	* Do not assume data transferred correctly if there are any error bits
2053	* set.
2054	*/
2055	if (cb_data.status & mmc_blk_stop_err_bits(brq: &mqrq->brq)) {
2056	mqrq->brq.data.bytes_xfered = 0;
2057	err = err ? err : -EIO;
2058	}
2059
2060	/* Copy the exception bit so it will be seen later on */
2061	if (mmc_card_mmc(card) && cb_data.status & R1_EXCEPTION_EVENT)
2062	mqrq->brq.cmd.resp[0] |= R1_EXCEPTION_EVENT;
2063
2064	return err;
2065	}
2066
2067	static inline void mmc_blk_rw_reset_success(struct mmc_queue *mq,
2068	struct request *req)
2069	{
2070	int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE;
2071
2072	mmc_blk_reset_success(md: mq->blkdata, type);
2073	}
2074
2075	static void mmc_blk_mq_complete_rq(struct mmc_queue *mq, struct request *req)
2076	{
2077	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
2078	unsigned int nr_bytes = mqrq->brq.data.bytes_xfered;
2079
2080	if (nr_bytes) {
2081	if (blk_update_request(rq: req, BLK_STS_OK, nr_bytes))
2082	blk_mq_requeue_request(rq: req, kick_requeue_list: true);
2083	else
2084	__blk_mq_end_request(rq: req, BLK_STS_OK);
2085	} else if (!blk_rq_bytes(rq: req)) {
2086	__blk_mq_end_request(rq: req, BLK_STS_IOERR);
2087	} else if (mqrq->retries++ < MMC_MAX_RETRIES) {
2088	blk_mq_requeue_request(rq: req, kick_requeue_list: true);
2089	} else {
2090	if (mmc_card_removed(mq->card))
2091	req->rq_flags |= RQF_QUIET;
2092	blk_mq_end_request(rq: req, BLK_STS_IOERR);
2093	}
2094	}
2095
2096	static bool mmc_blk_urgent_bkops_needed(struct mmc_queue *mq,
2097	struct mmc_queue_req *mqrq)
2098	{
2099	return mmc_card_mmc(mq->card) && !mmc_host_is_spi(mq->card->host) &&
2100	(mqrq->brq.cmd.resp[0] & R1_EXCEPTION_EVENT ||
2101	mqrq->brq.stop.resp[0] & R1_EXCEPTION_EVENT);
2102	}
2103
2104	static void mmc_blk_urgent_bkops(struct mmc_queue *mq,
2105	struct mmc_queue_req *mqrq)
2106	{
2107	if (mmc_blk_urgent_bkops_needed(mq, mqrq))
2108	mmc_run_bkops(card: mq->card);
2109	}
2110
2111	static void mmc_blk_hsq_req_done(struct mmc_request *mrq)
2112	{
2113	struct mmc_queue_req *mqrq =
2114	container_of(mrq, struct mmc_queue_req, brq.mrq);
2115	struct request *req = mmc_queue_req_to_req(mqr: mqrq);
2116	struct request_queue *q = req->q;
2117	struct mmc_queue *mq = q->queuedata;
2118	struct mmc_host *host = mq->card->host;
2119	unsigned long flags;
2120
2121	if (mmc_blk_rq_error(brq: &mqrq->brq) ||
2122	mmc_blk_urgent_bkops_needed(mq, mqrq)) {
2123	spin_lock_irqsave(&mq->lock, flags);
2124	mq->recovery_needed = true;
2125	mq->recovery_req = req;
2126	spin_unlock_irqrestore(lock: &mq->lock, flags);
2127
2128	host->cqe_ops->cqe_recovery_start(host);
2129
2130	schedule_work(work: &mq->recovery_work);
2131	return;
2132	}
2133
2134	mmc_blk_rw_reset_success(mq, req);
2135
2136	/*
2137	* Block layer timeouts race with completions which means the normal
2138	* completion path cannot be used during recovery.
2139	*/
2140	if (mq->in_recovery)
2141	mmc_blk_cqe_complete_rq(mq, req);
2142	else if (likely(!blk_should_fake_timeout(req->q)))
2143	blk_mq_complete_request(rq: req);
2144	}
2145
2146	void mmc_blk_mq_complete(struct request *req)
2147	{
2148	struct mmc_queue *mq = req->q->queuedata;
2149	struct mmc_host *host = mq->card->host;
2150
2151	if (host->cqe_enabled)
2152	mmc_blk_cqe_complete_rq(mq, req);
2153	else if (likely(!blk_should_fake_timeout(req->q)))
2154	mmc_blk_mq_complete_rq(mq, req);
2155	}
2156
2157	static void mmc_blk_mq_poll_completion(struct mmc_queue *mq,
2158	struct request *req)
2159	{
2160	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
2161	struct mmc_host *host = mq->card->host;
2162
2163	if (mmc_blk_rq_error(brq: &mqrq->brq) ||
2164	mmc_blk_card_busy(card: mq->card, req)) {
2165	mmc_blk_mq_rw_recovery(mq, req);
2166	} else {
2167	mmc_blk_rw_reset_success(mq, req);
2168	mmc_retune_release(host);
2169	}
2170
2171	mmc_blk_urgent_bkops(mq, mqrq);
2172	}
2173
2174	static void mmc_blk_mq_dec_in_flight(struct mmc_queue *mq, enum mmc_issue_type issue_type)
2175	{
2176	unsigned long flags;
2177	bool put_card;
2178
2179	spin_lock_irqsave(&mq->lock, flags);
2180
2181	mq->in_flight[issue_type] -= 1;
2182
2183	put_card = (mmc_tot_in_flight(mq) == 0);
2184
2185	spin_unlock_irqrestore(lock: &mq->lock, flags);
2186
2187	if (put_card)
2188	mmc_put_card(card: mq->card, ctx: &mq->ctx);
2189	}
2190
2191	static void mmc_blk_mq_post_req(struct mmc_queue *mq, struct request *req,
2192	bool can_sleep)
2193	{
2194	enum mmc_issue_type issue_type = mmc_issue_type(mq, req);
2195	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
2196	struct mmc_request *mrq = &mqrq->brq.mrq;
2197	struct mmc_host *host = mq->card->host;
2198
2199	mmc_post_req(host, mrq, err: 0);
2200
2201	/*
2202	* Block layer timeouts race with completions which means the normal
2203	* completion path cannot be used during recovery.
2204	*/
2205	if (mq->in_recovery) {
2206	mmc_blk_mq_complete_rq(mq, req);
2207	} else if (likely(!blk_should_fake_timeout(req->q))) {
2208	if (can_sleep)
2209	blk_mq_complete_request_direct(rq: req, complete: mmc_blk_mq_complete);
2210	else
2211	blk_mq_complete_request(rq: req);
2212	}
2213
2214	mmc_blk_mq_dec_in_flight(mq, issue_type);
2215	}
2216
2217	void mmc_blk_mq_recovery(struct mmc_queue *mq)
2218	{
2219	struct request *req = mq->recovery_req;
2220	struct mmc_host *host = mq->card->host;
2221	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
2222
2223	mq->recovery_req = NULL;
2224	mq->rw_wait = false;
2225
2226	if (mmc_blk_rq_error(brq: &mqrq->brq)) {
2227	mmc_retune_hold_now(host);
2228	mmc_blk_mq_rw_recovery(mq, req);
2229	}
2230
2231	mmc_blk_urgent_bkops(mq, mqrq);
2232
2233	mmc_blk_mq_post_req(mq, req, can_sleep: true);
2234	}
2235
2236	static void mmc_blk_mq_complete_prev_req(struct mmc_queue *mq,
2237	struct request **prev_req)
2238	{
2239	if (mmc_host_can_done_complete(host: mq->card->host))
2240	return;
2241
2242	mutex_lock(&mq->complete_lock);
2243
2244	if (!mq->complete_req)
2245	goto out_unlock;
2246
2247	mmc_blk_mq_poll_completion(mq, req: mq->complete_req);
2248
2249	if (prev_req)
2250	*prev_req = mq->complete_req;
2251	else
2252	mmc_blk_mq_post_req(mq, req: mq->complete_req, can_sleep: true);
2253
2254	mq->complete_req = NULL;
2255
2256	out_unlock:
2257	mutex_unlock(lock: &mq->complete_lock);
2258	}
2259
2260	void mmc_blk_mq_complete_work(struct work_struct *work)
2261	{
2262	struct mmc_queue *mq = container_of(work, struct mmc_queue,
2263	complete_work);
2264
2265	mmc_blk_mq_complete_prev_req(mq, NULL);
2266	}
2267
2268	static void mmc_blk_mq_req_done(struct mmc_request *mrq)
2269	{
2270	struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
2271	brq.mrq);
2272	struct request *req = mmc_queue_req_to_req(mqr: mqrq);
2273	struct request_queue *q = req->q;
2274	struct mmc_queue *mq = q->queuedata;
2275	struct mmc_host *host = mq->card->host;
2276	unsigned long flags;
2277
2278	if (!mmc_host_can_done_complete(host)) {
2279	bool waiting;
2280
2281	/*
2282	* We cannot complete the request in this context, so record
2283	* that there is a request to complete, and that a following
2284	* request does not need to wait (although it does need to
2285	* complete complete_req first).
2286	*/
2287	spin_lock_irqsave(&mq->lock, flags);
2288	mq->complete_req = req;
2289	mq->rw_wait = false;
2290	waiting = mq->waiting;
2291	spin_unlock_irqrestore(lock: &mq->lock, flags);
2292
2293	/*
2294	* If 'waiting' then the waiting task will complete this
2295	* request, otherwise queue a work to do it. Note that
2296	* complete_work may still race with the dispatch of a following
2297	* request.
2298	*/
2299	if (waiting)
2300	wake_up(&mq->wait);
2301	else
2302	queue_work(wq: mq->card->complete_wq, work: &mq->complete_work);
2303
2304	return;
2305	}
2306
2307	/* Take the recovery path for errors or urgent background operations */
2308	if (mmc_blk_rq_error(brq: &mqrq->brq) ||
2309	mmc_blk_urgent_bkops_needed(mq, mqrq)) {
2310	spin_lock_irqsave(&mq->lock, flags);
2311	mq->recovery_needed = true;
2312	mq->recovery_req = req;
2313	spin_unlock_irqrestore(lock: &mq->lock, flags);
2314	wake_up(&mq->wait);
2315	schedule_work(work: &mq->recovery_work);
2316	return;
2317	}
2318
2319	mmc_blk_rw_reset_success(mq, req);
2320
2321	mq->rw_wait = false;
2322	wake_up(&mq->wait);
2323
2324	/* context unknown */
2325	mmc_blk_mq_post_req(mq, req, can_sleep: false);
2326	}
2327
2328	static bool mmc_blk_rw_wait_cond(struct mmc_queue *mq, int *err)
2329	{
2330	unsigned long flags;
2331	bool done;
2332
2333	/*
2334	* Wait while there is another request in progress, but not if recovery
2335	* is needed. Also indicate whether there is a request waiting to start.
2336	*/
2337	spin_lock_irqsave(&mq->lock, flags);
2338	if (mq->recovery_needed) {
2339	*err = -EBUSY;
2340	done = true;
2341	} else {
2342	done = !mq->rw_wait;
2343	}
2344	mq->waiting = !done;
2345	spin_unlock_irqrestore(lock: &mq->lock, flags);
2346
2347	return done;
2348	}
2349
2350	static int mmc_blk_rw_wait(struct mmc_queue *mq, struct request **prev_req)
2351	{
2352	int err = 0;
2353
2354	wait_event(mq->wait, mmc_blk_rw_wait_cond(mq, &err));
2355
2356	/* Always complete the previous request if there is one */
2357	mmc_blk_mq_complete_prev_req(mq, prev_req);
2358
2359	return err;
2360	}
2361
2362	static int mmc_blk_mq_issue_rw_rq(struct mmc_queue *mq,
2363	struct request *req)
2364	{
2365	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(rq: req);
2366	struct mmc_host *host = mq->card->host;
2367	struct request *prev_req = NULL;
2368	int err = 0;
2369
2370	mmc_blk_rw_rq_prep(mqrq, card: mq->card, recovery_mode: 0, mq);
2371
2372	mqrq->brq.mrq.done = mmc_blk_mq_req_done;
2373
2374	mmc_pre_req(host, mrq: &mqrq->brq.mrq);
2375
2376	err = mmc_blk_rw_wait(mq, prev_req: &prev_req);
2377	if (err)
2378	goto out_post_req;
2379
2380	mq->rw_wait = true;
2381
2382	err = mmc_start_request(host, mrq: &mqrq->brq.mrq);
2383
2384	if (prev_req)
2385	mmc_blk_mq_post_req(mq, req: prev_req, can_sleep: true);
2386
2387	if (err)
2388	mq->rw_wait = false;
2389
2390	/* Release re-tuning here where there is no synchronization required */
2391	if (err || mmc_host_can_done_complete(host))
2392	mmc_retune_release(host);
2393
2394	out_post_req:
2395	if (err)
2396	mmc_post_req(host, mrq: &mqrq->brq.mrq, err);
2397
2398	return err;
2399	}
2400
2401	static int mmc_blk_wait_for_idle(struct mmc_queue *mq, struct mmc_host *host)
2402	{
2403	if (host->cqe_enabled)
2404	return host->cqe_ops->cqe_wait_for_idle(host);
2405
2406	return mmc_blk_rw_wait(mq, NULL);
2407	}
2408
2409	enum mmc_issued mmc_blk_mq_issue_rq(struct mmc_queue *mq, struct request *req)
2410	{
2411	struct mmc_blk_data *md = mq->blkdata;
2412	struct mmc_card *card = md->queue.card;
2413	struct mmc_host *host = card->host;
2414	int ret;
2415
2416	ret = mmc_blk_part_switch(card, part_type: md->part_type);
2417	if (ret)
2418	return MMC_REQ_FAILED_TO_START;
2419
2420	switch (mmc_issue_type(mq, req)) {
2421	case MMC_ISSUE_SYNC:
2422	ret = mmc_blk_wait_for_idle(mq, host);
2423	if (ret)
2424	return MMC_REQ_BUSY;
2425	switch (req_op(req)) {
2426	case REQ_OP_DRV_IN:
2427	case REQ_OP_DRV_OUT:
2428	mmc_blk_issue_drv_op(mq, req);
2429	break;
2430	case REQ_OP_DISCARD:
2431	mmc_blk_issue_discard_rq(mq, req);
2432	break;
2433	case REQ_OP_SECURE_ERASE:
2434	mmc_blk_issue_secdiscard_rq(mq, req);
2435	break;
2436	case REQ_OP_WRITE_ZEROES:
2437	mmc_blk_issue_trim_rq(mq, req);
2438	break;
2439	case REQ_OP_FLUSH:
2440	mmc_blk_issue_flush(mq, req);
2441	break;
2442	default:
2443	WARN_ON_ONCE(1);
2444	return MMC_REQ_FAILED_TO_START;
2445	}
2446	return MMC_REQ_FINISHED;
2447	case MMC_ISSUE_DCMD:
2448	case MMC_ISSUE_ASYNC:
2449	switch (req_op(req)) {
2450	case REQ_OP_FLUSH:
2451	if (!mmc_cache_enabled(host)) {
2452	blk_mq_end_request(rq: req, BLK_STS_OK);
2453	return MMC_REQ_FINISHED;
2454	}
2455	ret = mmc_blk_cqe_issue_flush(mq, req);
2456	break;
2457	case REQ_OP_WRITE:
2458	card->written_flag = true;
2459	fallthrough;
2460	case REQ_OP_READ:
2461	if (host->cqe_enabled)
2462	ret = mmc_blk_cqe_issue_rw_rq(mq, req);
2463	else
2464	ret = mmc_blk_mq_issue_rw_rq(mq, req);
2465	break;
2466	default:
2467	WARN_ON_ONCE(1);
2468	ret = -EINVAL;
2469	}
2470	if (!ret)
2471	return MMC_REQ_STARTED;
2472	return ret == -EBUSY ? MMC_REQ_BUSY : MMC_REQ_FAILED_TO_START;
2473	default:
2474	WARN_ON_ONCE(1);
2475	return MMC_REQ_FAILED_TO_START;
2476	}
2477	}
2478
2479	static inline int mmc_blk_readonly(struct mmc_card *card)
2480	{
2481	return mmc_card_readonly(card) ||
2482	!(card->csd.cmdclass & CCC_BLOCK_WRITE);
2483	}
2484
2485	/*
2486	* Search for a declared partitions node for the disk in mmc-card related node.
2487	*
2488	* This is to permit support for partition table defined in DT in special case
2489	* where a partition table is not written in the disk and is expected to be
2490	* passed from the running system.
2491	*
2492	* For the user disk, "partitions" node is searched.
2493	* For the special HW disk, "partitions-" node with the appended name is used
2494	* following this conversion table (to adhere to JEDEC naming)
2495	* - boot0 -> partitions-boot1
2496	* - boot1 -> partitions-boot2
2497	* - gp0 -> partitions-gp1
2498	* - gp1 -> partitions-gp2
2499	* - gp2 -> partitions-gp3
2500	* - gp3 -> partitions-gp4
2501	*/
2502	static struct fwnode_handle *mmc_blk_get_partitions_node(struct device *mmc_dev,
2503	const char *subname)
2504	{
2505	const char *node_name = "partitions";
2506
2507	if (subname) {
2508	mmc_dev = mmc_dev->parent;
2509
2510	/*
2511	* Check if we are allocating a BOOT disk boot0/1 disk.
2512	* In DT we use the JEDEC naming boot1/2.
2513	*/
2514	if (!strcmp(subname, "boot0"))
2515	node_name = "partitions-boot1";
2516	if (!strcmp(subname, "boot1"))
2517	node_name = "partitions-boot2";
2518	/*
2519	* Check if we are allocating a GP disk gp0/1/2/3 disk.
2520	* In DT we use the JEDEC naming gp1/2/3/4.
2521	*/
2522	if (!strcmp(subname, "gp0"))
2523	node_name = "partitions-gp1";
2524	if (!strcmp(subname, "gp1"))
2525	node_name = "partitions-gp2";
2526	if (!strcmp(subname, "gp2"))
2527	node_name = "partitions-gp3";
2528	if (!strcmp(subname, "gp3"))
2529	node_name = "partitions-gp4";
2530	}
2531
2532	return device_get_named_child_node(dev: mmc_dev, childname: node_name);
2533	}
2534
2535	static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card,
2536	struct device *parent,
2537	sector_t size,
2538	bool default_ro,
2539	const char *subname,
2540	int area_type,
2541	unsigned int part_type)
2542	{
2543	struct fwnode_handle *disk_fwnode;
2544	struct mmc_blk_data *md;
2545	int devidx, ret;
2546	char cap_str[10];
2547	unsigned int features = 0;
2548
2549	devidx = ida_alloc_max(ida: &mmc_blk_ida, max: max_devices - 1, GFP_KERNEL);
2550	if (devidx < 0) {
2551	/*
2552	* We get -ENOSPC because there are no more any available
2553	* devidx. The reason may be that, either userspace haven't yet
2554	* unmounted the partitions, which postpones mmc_blk_release()
2555	* from being called, or the device has more partitions than
2556	* what we support.
2557	*/
2558	if (devidx == -ENOSPC)
2559	dev_err(mmc_dev(card->host),
2560	"no more device IDs available\n");
2561
2562	return ERR_PTR(error: devidx);
2563	}
2564
2565	md = kzalloc(sizeof(*md), GFP_KERNEL);
2566	if (!md) {
2567	ret = -ENOMEM;
2568	goto out;
2569	}
2570
2571	md->area_type = area_type;
2572
2573	/*
2574	* Set the read-only status based on the supported commands
2575	* and the write protect switch.
2576	*/
2577	md->read_only = mmc_blk_readonly(card);
2578
2579	if (mmc_host_can_cmd23(host: card->host) && mmc_card_can_cmd23(card))
2580	md->flags |= MMC_BLK_CMD23;
2581
2582	if (md->flags & MMC_BLK_CMD23 &&
2583	((card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN) ||
2584	card->ext_csd.rel_sectors)) {
2585	md->flags |= MMC_BLK_REL_WR;
2586	features |= (BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA);
2587	} else if (mmc_cache_enabled(host: card->host)) {
2588	features |= BLK_FEAT_WRITE_CACHE;
2589	}
2590
2591	md->disk = mmc_init_queue(mq: &md->queue, card, features);
2592	if (IS_ERR(ptr: md->disk)) {
2593	ret = PTR_ERR(ptr: md->disk);
2594	goto err_kfree;
2595	}
2596
2597	INIT_LIST_HEAD(list: &md->part);
2598	INIT_LIST_HEAD(list: &md->rpmbs);
2599	kref_init(kref: &md->kref);
2600
2601	md->queue.blkdata = md;
2602	md->part_type = part_type;
2603
2604	md->disk->major = MMC_BLOCK_MAJOR;
2605	md->disk->minors = perdev_minors;
2606	md->disk->first_minor = devidx * perdev_minors;
2607	md->disk->fops = &mmc_bdops;
2608	md->disk->private_data = md;
2609	md->parent = parent;
2610	set_disk_ro(disk: md->disk, read_only: md->read_only || default_ro);
2611	if (area_type & MMC_BLK_DATA_AREA_RPMB)
2612	md->disk->flags |= GENHD_FL_NO_PART;
2613
2614	/*
2615	* As discussed on lkml, GENHD_FL_REMOVABLE should:
2616	*
2617	* - be set for removable media with permanent block devices
2618	* - be unset for removable block devices with permanent media
2619	*
2620	* Since MMC block devices clearly fall under the second
2621	* case, we do not set GENHD_FL_REMOVABLE. Userspace
2622	* should use the block device creation/destruction hotplug
2623	* messages to tell when the card is present.
2624	*/
2625
2626	snprintf(buf: md->disk->disk_name, size: sizeof(md->disk->disk_name),
2627	fmt: "mmcblk%u%s", card->host->index, subname ? subname : "");
2628
2629	set_capacity(disk: md->disk, size);
2630
2631	string_get_size(size: (u64)size, blk_size: 512, units: STRING_UNITS_2,
2632	buf: cap_str, len: sizeof(cap_str));
2633	pr_info("%s: %s %s %s%s\n",
2634	md->disk->disk_name, mmc_card_id(card), mmc_card_name(card),
2635	cap_str, md->read_only ? " (ro)" : "");
2636
2637	/* used in ->open, must be set before add_disk: */
2638	if (area_type == MMC_BLK_DATA_AREA_MAIN)
2639	dev_set_drvdata(dev: &card->dev, data: md);
2640	disk_fwnode = mmc_blk_get_partitions_node(mmc_dev: parent, subname);
2641	ret = add_disk_fwnode(parent: md->parent, disk: md->disk, groups: mmc_disk_attr_groups,
2642	fwnode: disk_fwnode);
2643	if (ret)
2644	goto err_put_disk;
2645	return md;
2646
2647	err_put_disk:
2648	put_disk(disk: md->disk);
2649	blk_mq_free_tag_set(set: &md->queue.tag_set);
2650	err_kfree:
2651	kfree(objp: md);
2652	out:
2653	ida_free(&mmc_blk_ida, id: devidx);
2654	return ERR_PTR(error: ret);
2655	}
2656
2657	static struct mmc_blk_data *mmc_blk_alloc(struct mmc_card *card)
2658	{
2659	sector_t size;
2660
2661	if (!mmc_card_sd(card) && mmc_card_blockaddr(card)) {
2662	/*
2663	* The EXT_CSD sector count is in number or 512 byte
2664	* sectors.
2665	*/
2666	size = card->ext_csd.sectors;
2667	} else {
2668	/*
2669	* The CSD capacity field is in units of read_blkbits.
2670	* set_capacity takes units of 512 bytes.
2671	*/
2672	size = (typeof(sector_t))card->csd.capacity
2673	<< (card->csd.read_blkbits - 9);
2674	}
2675
2676	return mmc_blk_alloc_req(card, parent: &card->dev, size, default_ro: false, NULL,
2677	MMC_BLK_DATA_AREA_MAIN, part_type: 0);
2678	}
2679
2680	static int mmc_blk_alloc_part(struct mmc_card *card,
2681	struct mmc_blk_data *md,
2682	unsigned int part_type,
2683	sector_t size,
2684	bool default_ro,
2685	const char *subname,
2686	int area_type)
2687	{
2688	struct mmc_blk_data *part_md;
2689
2690	part_md = mmc_blk_alloc_req(card, disk_to_dev(md->disk), size, default_ro,
2691	subname, area_type, part_type);
2692	if (IS_ERR(ptr: part_md))
2693	return PTR_ERR(ptr: part_md);
2694	list_add(new: &part_md->part, head: &md->part);
2695
2696	return 0;
2697	}
2698
2699	/**
2700	* mmc_rpmb_ioctl() - ioctl handler for the RPMB chardev
2701	* @filp: the character device file
2702	* @cmd: the ioctl() command
2703	* @arg: the argument from userspace
2704	*
2705	* This will essentially just redirect the ioctl()s coming in over to
2706	* the main block device spawning the RPMB character device.
2707	*/
2708	static long mmc_rpmb_ioctl(struct file *filp, unsigned int cmd,
2709	unsigned long arg)
2710	{
2711	struct mmc_rpmb_data *rpmb = filp->private_data;
2712	int ret;
2713
2714	switch (cmd) {
2715	case MMC_IOC_CMD:
2716	ret = mmc_blk_ioctl_cmd(md: rpmb->md,
2717	ic_ptr: (struct mmc_ioc_cmd __user *)arg,
2718	rpmb);
2719	break;
2720	case MMC_IOC_MULTI_CMD:
2721	ret = mmc_blk_ioctl_multi_cmd(md: rpmb->md,
2722	user: (struct mmc_ioc_multi_cmd __user *)arg,
2723	rpmb);
2724	break;
2725	default:
2726	ret = -EINVAL;
2727	break;
2728	}
2729
2730	return ret;
2731	}
2732
2733	#ifdef CONFIG_COMPAT
2734	static long mmc_rpmb_ioctl_compat(struct file *filp, unsigned int cmd,
2735	unsigned long arg)
2736	{
2737	return mmc_rpmb_ioctl(filp, cmd, arg: (unsigned long)compat_ptr(uptr: arg));
2738	}
2739	#endif
2740
2741	static int mmc_rpmb_chrdev_open(struct inode *inode, struct file *filp)
2742	{
2743	struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev,
2744	struct mmc_rpmb_data, chrdev);
2745
2746	get_device(dev: &rpmb->dev);
2747	filp->private_data = rpmb;
2748
2749	return nonseekable_open(inode, filp);
2750	}
2751
2752	static int mmc_rpmb_chrdev_release(struct inode *inode, struct file *filp)
2753	{
2754	struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev,
2755	struct mmc_rpmb_data, chrdev);
2756
2757	put_device(dev: &rpmb->dev);
2758
2759	return 0;
2760	}
2761
2762	static const struct file_operations mmc_rpmb_fileops = {
2763	.release = mmc_rpmb_chrdev_release,
2764	.open = mmc_rpmb_chrdev_open,
2765	.owner = THIS_MODULE,
2766	.unlocked_ioctl = mmc_rpmb_ioctl,
2767	#ifdef CONFIG_COMPAT
2768	.compat_ioctl = mmc_rpmb_ioctl_compat,
2769	#endif
2770	};
2771
2772	static void mmc_blk_rpmb_device_release(struct device *dev)
2773	{
2774	struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev);
2775
2776	rpmb_dev_unregister(rdev: rpmb->rdev);
2777	mmc_blk_put(md: rpmb->md);
2778	ida_free(&mmc_rpmb_ida, id: rpmb->id);
2779	kfree(objp: rpmb);
2780	}
2781
2782	static void free_idata(struct mmc_blk_ioc_data **idata, unsigned int cmd_count)
2783	{
2784	unsigned int n;
2785
2786	for (n = 0; n < cmd_count; n++)
2787	kfree(objp: idata[n]);
2788	kfree(objp: idata);
2789	}
2790
2791	static struct mmc_blk_ioc_data **alloc_idata(struct mmc_rpmb_data *rpmb,
2792	unsigned int cmd_count)
2793	{
2794	struct mmc_blk_ioc_data **idata;
2795	unsigned int n;
2796
2797	idata = kcalloc(cmd_count, sizeof(*idata), GFP_KERNEL);
2798	if (!idata)
2799	return NULL;
2800
2801	for (n = 0; n < cmd_count; n++) {
2802	idata[n] = kcalloc(1, sizeof(**idata), GFP_KERNEL);
2803	if (!idata[n]) {
2804	free_idata(idata, cmd_count: n);
2805	return NULL;
2806	}
2807	idata[n]->rpmb = rpmb;
2808	}
2809
2810	return idata;
2811	}
2812
2813	static void set_idata(struct mmc_blk_ioc_data *idata, u32 opcode,
2814	int write_flag, u8 *buf, unsigned int buf_bytes)
2815	{
2816	/*
2817	* The size of an RPMB frame must match what's expected by the
2818	* hardware.
2819	*/
2820	static_assert(!CHECK_SIZE_NEQ(512), "RPMB frame size must be 512 bytes");
2821
2822	idata->ic.opcode = opcode;
2823	idata->ic.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
2824	idata->ic.write_flag = write_flag;
2825	idata->ic.blksz = RPMB_FRAME_SIZE;
2826	idata->ic.blocks = buf_bytes / idata->ic.blksz;
2827	idata->buf = buf;
2828	idata->buf_bytes = buf_bytes;
2829	}
2830
2831	static int mmc_route_rpmb_frames(struct device *dev, u8 *req,
2832	unsigned int req_len, u8 *resp,
2833	unsigned int resp_len)
2834	{
2835	struct rpmb_frame *frm = (struct rpmb_frame *)req;
2836	struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev);
2837	struct mmc_blk_data *md = rpmb->md;
2838	struct mmc_blk_ioc_data **idata;
2839	struct mmc_queue_req *mq_rq;
2840	unsigned int cmd_count;
2841	struct request *rq;
2842	u16 req_type;
2843	bool write;
2844	int ret;
2845
2846	if (IS_ERR(ptr: md->queue.card))
2847	return PTR_ERR(ptr: md->queue.card);
2848
2849	if (req_len < RPMB_FRAME_SIZE)
2850	return -EINVAL;
2851
2852	req_type = be16_to_cpu(frm->req_resp);
2853	switch (req_type) {
2854	case RPMB_PROGRAM_KEY:
2855	if (CHECK_SIZE_NEQ(req_len) || CHECK_SIZE_NEQ(resp_len))
2856	return -EINVAL;
2857	write = true;
2858	break;
2859	case RPMB_GET_WRITE_COUNTER:
2860	if (CHECK_SIZE_NEQ(req_len) || CHECK_SIZE_NEQ(resp_len))
2861	return -EINVAL;
2862	write = false;
2863	break;
2864	case RPMB_WRITE_DATA:
2865	if (!CHECK_SIZE_ALIGNED(req_len) || CHECK_SIZE_NEQ(resp_len))
2866	return -EINVAL;
2867	write = true;
2868	break;
2869	case RPMB_READ_DATA:
2870	if (CHECK_SIZE_NEQ(req_len) || !CHECK_SIZE_ALIGNED(resp_len))
2871	return -EINVAL;
2872	write = false;
2873	break;
2874	default:
2875	return -EINVAL;
2876	}
2877
2878	/* Write operations require 3 commands, read operations require 2 */
2879	cmd_count = write ? 3 : 2;
2880
2881	idata = alloc_idata(rpmb, cmd_count);
2882	if (!idata)
2883	return -ENOMEM;
2884
2885	if (write) {
2886	struct rpmb_frame *resp_frm = (struct rpmb_frame *)resp;
2887
2888	/* Send write request frame(s) */
2889	set_idata(idata: idata[0], MMC_WRITE_MULTIPLE_BLOCK,
2890	write_flag: 1 | MMC_CMD23_ARG_REL_WR, buf: req, buf_bytes: req_len);
2891
2892	/* Send result request frame */
2893	memset(resp_frm, 0, RPMB_FRAME_SIZE);
2894	resp_frm->req_resp = cpu_to_be16(RPMB_RESULT_READ);
2895	set_idata(idata: idata[1], MMC_WRITE_MULTIPLE_BLOCK, write_flag: 1, buf: resp,
2896	buf_bytes: resp_len);
2897
2898	/* Read response frame */
2899	set_idata(idata: idata[2], MMC_READ_MULTIPLE_BLOCK, write_flag: 0, buf: resp, buf_bytes: resp_len);
2900	} else {
2901	/* Send write request frame(s) */
2902	set_idata(idata: idata[0], MMC_WRITE_MULTIPLE_BLOCK, write_flag: 1, buf: req, buf_bytes: req_len);
2903
2904	/* Read response frame */
2905	set_idata(idata: idata[1], MMC_READ_MULTIPLE_BLOCK, write_flag: 0, buf: resp, buf_bytes: resp_len);
2906	}
2907
2908	rq = blk_mq_alloc_request(q: md->queue.queue, opf: REQ_OP_DRV_OUT, flags: 0);
2909	if (IS_ERR(ptr: rq)) {
2910	ret = PTR_ERR(ptr: rq);
2911	goto out;
2912	}
2913
2914	mq_rq = req_to_mmc_queue_req(rq);
2915	mq_rq->drv_op = MMC_DRV_OP_IOCTL_RPMB;
2916	mq_rq->drv_op_result = -EIO;
2917	mq_rq->drv_op_data = idata;
2918	mq_rq->ioc_count = cmd_count;
2919	blk_execute_rq(rq, at_head: false);
2920	ret = req_to_mmc_queue_req(rq)->drv_op_result;
2921
2922	blk_mq_free_request(rq);
2923
2924	out:
2925	free_idata(idata, cmd_count);
2926	return ret;
2927	}
2928
2929	static int mmc_blk_alloc_rpmb_part(struct mmc_card *card,
2930	struct mmc_blk_data *md,
2931	unsigned int part_index,
2932	sector_t size,
2933	const char *subname)
2934	{
2935	int devidx, ret;
2936	char rpmb_name[DISK_NAME_LEN];
2937	char cap_str[10];
2938	struct mmc_rpmb_data *rpmb;
2939
2940	/* This creates the minor number for the RPMB char device */
2941	devidx = ida_alloc_max(ida: &mmc_rpmb_ida, max: max_devices - 1, GFP_KERNEL);
2942	if (devidx < 0)
2943	return devidx;
2944
2945	rpmb = kzalloc(sizeof(*rpmb), GFP_KERNEL);
2946	if (!rpmb) {
2947	ida_free(&mmc_rpmb_ida, id: devidx);
2948	return -ENOMEM;
2949	}
2950
2951	snprintf(buf: rpmb_name, size: sizeof(rpmb_name),
2952	fmt: "mmcblk%u%s", card->host->index, subname ? subname : "");
2953
2954	rpmb->id = devidx;
2955	rpmb->part_index = part_index;
2956	rpmb->dev.init_name = rpmb_name;
2957	rpmb->dev.bus = &mmc_rpmb_bus_type;
2958	rpmb->dev.devt = MKDEV(MAJOR(mmc_rpmb_devt), rpmb->id);
2959	rpmb->dev.parent = &card->dev;
2960	rpmb->dev.release = mmc_blk_rpmb_device_release;
2961	device_initialize(dev: &rpmb->dev);
2962	dev_set_drvdata(dev: &rpmb->dev, data: rpmb);
2963	mmc_blk_get(disk: md->disk);
2964	rpmb->md = md;
2965
2966	cdev_init(&rpmb->chrdev, &mmc_rpmb_fileops);
2967	rpmb->chrdev.owner = THIS_MODULE;
2968	ret = cdev_device_add(cdev: &rpmb->chrdev, dev: &rpmb->dev);
2969	if (ret) {
2970	pr_err("%s: could not add character device\n", rpmb_name);
2971	goto out_put_device;
2972	}
2973
2974	list_add(new: &rpmb->node, head: &md->rpmbs);
2975
2976	string_get_size(size: (u64)size, blk_size: 512, units: STRING_UNITS_2,
2977	buf: cap_str, len: sizeof(cap_str));
2978
2979	pr_info("%s: %s %s %s, chardev (%d:%d)\n",
2980	rpmb_name, mmc_card_id(card), mmc_card_name(card), cap_str,
2981	MAJOR(mmc_rpmb_devt), rpmb->id);
2982
2983	return 0;
2984
2985	out_put_device:
2986	put_device(dev: &rpmb->dev);
2987	return ret;
2988	}
2989
2990	static void mmc_blk_remove_rpmb_part(struct mmc_rpmb_data *rpmb)
2991
2992	{
2993	cdev_device_del(cdev: &rpmb->chrdev, dev: &rpmb->dev);
2994	put_device(dev: &rpmb->dev);
2995	}
2996
2997	/* MMC Physical partitions consist of two boot partitions and
2998	* up to four general purpose partitions.
2999	* For each partition enabled in EXT_CSD a block device will be allocatedi
3000	* to provide access to the partition.
3001	*/
3002
3003	static int mmc_blk_alloc_parts(struct mmc_card *card, struct mmc_blk_data *md)
3004	{
3005	int idx, ret;
3006
3007	if (!mmc_card_mmc(card))
3008	return 0;
3009
3010	for (idx = 0; idx < card->nr_parts; idx++) {
3011	if (card->part[idx].area_type & MMC_BLK_DATA_AREA_RPMB) {
3012	/*
3013	* RPMB partitions does not provide block access, they
3014	* are only accessed using ioctl():s. Thus create
3015	* special RPMB block devices that do not have a
3016	* backing block queue for these.
3017	*/
3018	ret = mmc_blk_alloc_rpmb_part(card, md,
3019	part_index: card->part[idx].part_cfg,
3020	size: card->part[idx].size >> 9,
3021	subname: card->part[idx].name);
3022	if (ret)
3023	return ret;
3024	} else if (card->part[idx].size) {
3025	ret = mmc_blk_alloc_part(card, md,
3026	part_type: card->part[idx].part_cfg,
3027	size: card->part[idx].size >> 9,
3028	default_ro: card->part[idx].force_ro,
3029	subname: card->part[idx].name,
3030	area_type: card->part[idx].area_type);
3031	if (ret)
3032	return ret;
3033	}
3034	}
3035
3036	return 0;
3037	}
3038
3039	static void mmc_blk_remove_req(struct mmc_blk_data *md)
3040	{
3041	/*
3042	* Flush remaining requests and free queues. It is freeing the queue
3043	* that stops new requests from being accepted.
3044	*/
3045	del_gendisk(gp: md->disk);
3046	mmc_cleanup_queue(&md->queue);
3047	mmc_blk_put(md);
3048	}
3049
3050	static void mmc_blk_remove_parts(struct mmc_card *card,
3051	struct mmc_blk_data *md)
3052	{
3053	struct list_head *pos, *q;
3054	struct mmc_blk_data *part_md;
3055	struct mmc_rpmb_data *rpmb;
3056
3057	/* Remove RPMB partitions */
3058	list_for_each_safe(pos, q, &md->rpmbs) {
3059	rpmb = list_entry(pos, struct mmc_rpmb_data, node);
3060	list_del(entry: pos);
3061	mmc_blk_remove_rpmb_part(rpmb);
3062	}
3063	/* Remove block partitions */
3064	list_for_each_safe(pos, q, &md->part) {
3065	part_md = list_entry(pos, struct mmc_blk_data, part);
3066	list_del(entry: pos);
3067	mmc_blk_remove_req(md: part_md);
3068	}
3069	}
3070
3071	#ifdef CONFIG_DEBUG_FS
3072
3073	static int mmc_dbg_card_status_get(void *data, u64 *val)
3074	{
3075	struct mmc_card *card = data;
3076	struct mmc_blk_data *md = dev_get_drvdata(dev: &card->dev);
3077	struct mmc_queue *mq = &md->queue;
3078	struct request *req;
3079	int ret;
3080
3081	/* Ask the block layer about the card status */
3082	req = blk_mq_alloc_request(q: mq->queue, opf: REQ_OP_DRV_IN, flags: 0);
3083	if (IS_ERR(ptr: req))
3084	return PTR_ERR(ptr: req);
3085	req_to_mmc_queue_req(rq: req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS;
3086	req_to_mmc_queue_req(rq: req)->drv_op_result = -EIO;
3087	blk_execute_rq(rq: req, at_head: false);
3088	ret = req_to_mmc_queue_req(rq: req)->drv_op_result;
3089	if (ret >= 0) {
3090	*val = ret;
3091	ret = 0;
3092	}
3093	blk_mq_free_request(rq: req);
3094
3095	return ret;
3096	}
3097	DEFINE_DEBUGFS_ATTRIBUTE(mmc_dbg_card_status_fops, mmc_dbg_card_status_get,
3098	NULL, "%08llx\n");
3099
3100	/* That is two digits * 512 + 1 for newline */
3101	#define EXT_CSD_STR_LEN 1025
3102
3103	static int mmc_ext_csd_open(struct inode *inode, struct file *filp)
3104	{
3105	struct mmc_card *card = inode->i_private;
3106	struct mmc_blk_data *md = dev_get_drvdata(dev: &card->dev);
3107	struct mmc_queue *mq = &md->queue;
3108	struct request *req;
3109	char *buf;
3110	ssize_t n = 0;
3111	u8 *ext_csd;
3112	int err, i;
3113
3114	buf = kmalloc(EXT_CSD_STR_LEN + 1, GFP_KERNEL);
3115	if (!buf)
3116	return -ENOMEM;
3117
3118	/* Ask the block layer for the EXT CSD */
3119	req = blk_mq_alloc_request(q: mq->queue, opf: REQ_OP_DRV_IN, flags: 0);
3120	if (IS_ERR(ptr: req)) {
3121	err = PTR_ERR(ptr: req);
3122	goto out_free;
3123	}
3124	req_to_mmc_queue_req(rq: req)->drv_op = MMC_DRV_OP_GET_EXT_CSD;
3125	req_to_mmc_queue_req(rq: req)->drv_op_result = -EIO;
3126	req_to_mmc_queue_req(rq: req)->drv_op_data = &ext_csd;
3127	blk_execute_rq(rq: req, at_head: false);
3128	err = req_to_mmc_queue_req(rq: req)->drv_op_result;
3129	blk_mq_free_request(rq: req);
3130	if (err) {
3131	pr_err("FAILED %d\n", err);
3132	goto out_free;
3133	}
3134
3135	for (i = 0; i < 512; i++)
3136	n += sprintf(buf: buf + n, fmt: "%02x", ext_csd[i]);
3137	n += sprintf(buf: buf + n, fmt: "\n");
3138
3139	if (n != EXT_CSD_STR_LEN) {
3140	err = -EINVAL;
3141	kfree(objp: ext_csd);
3142	goto out_free;
3143	}
3144
3145	filp->private_data = buf;
3146	kfree(objp: ext_csd);
3147	return 0;
3148
3149	out_free:
3150	kfree(objp: buf);
3151	return err;
3152	}
3153
3154	static ssize_t mmc_ext_csd_read(struct file *filp, char __user *ubuf,
3155	size_t cnt, loff_t *ppos)
3156	{
3157	char *buf = filp->private_data;
3158
3159	return simple_read_from_buffer(to: ubuf, count: cnt, ppos,
3160	from: buf, EXT_CSD_STR_LEN);
3161	}
3162
3163	static int mmc_ext_csd_release(struct inode *inode, struct file *file)
3164	{
3165	kfree(objp: file->private_data);
3166	return 0;
3167	}
3168
3169	static const struct file_operations mmc_dbg_ext_csd_fops = {
3170	.open = mmc_ext_csd_open,
3171	.read = mmc_ext_csd_read,
3172	.release = mmc_ext_csd_release,
3173	.llseek = default_llseek,
3174	};
3175
3176	static void mmc_blk_add_debugfs(struct mmc_card *card, struct mmc_blk_data *md)
3177	{
3178	struct dentry *root;
3179
3180	if (!card->debugfs_root)
3181	return;
3182
3183	root = card->debugfs_root;
3184
3185	if (mmc_card_mmc(card) || mmc_card_sd(card)) {
3186	md->status_dentry =
3187	debugfs_create_file_unsafe(name: "status", mode: 0400, parent: root,
3188	data: card,
3189	fops: &mmc_dbg_card_status_fops);
3190	}
3191
3192	if (mmc_card_mmc(card)) {
3193	md->ext_csd_dentry =
3194	debugfs_create_file("ext_csd", 0400, root, card,
3195	&mmc_dbg_ext_csd_fops);
3196	}
3197	}
3198
3199	static void mmc_blk_remove_debugfs(struct mmc_card *card,
3200	struct mmc_blk_data *md)
3201	{
3202	if (!card->debugfs_root)
3203	return;
3204
3205	debugfs_remove(dentry: md->status_dentry);
3206	md->status_dentry = NULL;
3207
3208	debugfs_remove(dentry: md->ext_csd_dentry);
3209	md->ext_csd_dentry = NULL;
3210	}
3211
3212	#else
3213
3214	static void mmc_blk_add_debugfs(struct mmc_card *card, struct mmc_blk_data *md)
3215	{
3216	}
3217
3218	static void mmc_blk_remove_debugfs(struct mmc_card *card,
3219	struct mmc_blk_data *md)
3220	{
3221	}
3222
3223	#endif /* CONFIG_DEBUG_FS */
3224
3225	static void mmc_blk_rpmb_add(struct mmc_card *card)
3226	{
3227	struct mmc_blk_data *md = dev_get_drvdata(dev: &card->dev);
3228	struct mmc_rpmb_data *rpmb;
3229	struct rpmb_dev *rdev;
3230	unsigned int n;
3231	u32 cid[4];
3232	struct rpmb_descr descr = {
3233	.type = RPMB_TYPE_EMMC,
3234	.route_frames = mmc_route_rpmb_frames,
3235	.reliable_wr_count = card->ext_csd.enhanced_rpmb_supported ?
3236	2 : 32,
3237	.capacity = card->ext_csd.raw_rpmb_size_mult,
3238	.dev_id = (void *)cid,
3239	.dev_id_len = sizeof(cid),
3240	};
3241
3242	/*
3243	* Provice CID as an octet array. The CID needs to be interpreted
3244	* when used as input to derive the RPMB key since some fields
3245	* will change due to firmware updates.
3246	*/
3247	for (n = 0; n < 4; n++)
3248	cid[n] = be32_to_cpu((__force __be32)card->raw_cid[n]);
3249
3250	list_for_each_entry(rpmb, &md->rpmbs, node) {
3251	rdev = rpmb_dev_register(dev: &rpmb->dev, descr: &descr);
3252	if (IS_ERR(ptr: rdev)) {
3253	pr_warn("%s: could not register RPMB device\n",
3254	dev_name(&rpmb->dev));
3255	continue;
3256	}
3257	rpmb->rdev = rdev;
3258	}
3259	}
3260
3261	static int mmc_blk_probe(struct mmc_card *card)
3262	{
3263	struct mmc_blk_data *md;
3264	int ret = 0;
3265
3266	/*
3267	* Check that the card supports the command class(es) we need.
3268	*/
3269	if (!(card->csd.cmdclass & CCC_BLOCK_READ))
3270	return -ENODEV;
3271
3272	mmc_fixup_device(card, table: mmc_blk_fixups);
3273
3274	card->complete_wq = alloc_workqueue("mmc_complete",
3275	WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU,
3276	0);
3277	if (!card->complete_wq) {
3278	pr_err("Failed to create mmc completion workqueue");
3279	return -ENOMEM;
3280	}
3281
3282	md = mmc_blk_alloc(card);
3283	if (IS_ERR(ptr: md)) {
3284	ret = PTR_ERR(ptr: md);
3285	goto out_free;
3286	}
3287
3288	ret = mmc_blk_alloc_parts(card, md);
3289	if (ret)
3290	goto out;
3291
3292	/* Add two debugfs entries */
3293	mmc_blk_add_debugfs(card, md);
3294
3295	pm_runtime_set_autosuspend_delay(dev: &card->dev, delay: 3000);
3296	pm_runtime_use_autosuspend(dev: &card->dev);
3297
3298	/*
3299	* Don't enable runtime PM for SD-combo cards here. Leave that
3300	* decision to be taken during the SDIO init sequence instead.
3301	*/
3302	if (!mmc_card_sd_combo(card)) {
3303	pm_runtime_set_active(dev: &card->dev);
3304	pm_runtime_enable(dev: &card->dev);
3305	}
3306
3307	mmc_blk_rpmb_add(card);
3308
3309	return 0;
3310
3311	out:
3312	mmc_blk_remove_parts(card, md);
3313	mmc_blk_remove_req(md);
3314	out_free:
3315	destroy_workqueue(wq: card->complete_wq);
3316	return ret;
3317	}
3318
3319	static void mmc_blk_remove(struct mmc_card *card)
3320	{
3321	struct mmc_blk_data *md = dev_get_drvdata(dev: &card->dev);
3322
3323	mmc_blk_remove_debugfs(card, md);
3324	mmc_blk_remove_parts(card, md);
3325	pm_runtime_get_sync(dev: &card->dev);
3326	if (md->part_curr != md->part_type) {
3327	mmc_claim_host(host: card->host);
3328	mmc_blk_part_switch(card, part_type: md->part_type);
3329	mmc_release_host(host: card->host);
3330	}
3331	if (!mmc_card_sd_combo(card))
3332	pm_runtime_disable(dev: &card->dev);
3333	pm_runtime_put_noidle(dev: &card->dev);
3334	mmc_blk_remove_req(md);
3335	destroy_workqueue(wq: card->complete_wq);
3336	}
3337
3338	static int _mmc_blk_suspend(struct mmc_card *card)
3339	{
3340	struct mmc_blk_data *part_md;
3341	struct mmc_blk_data *md = dev_get_drvdata(dev: &card->dev);
3342
3343	if (md) {
3344	mmc_queue_suspend(&md->queue);
3345	list_for_each_entry(part_md, &md->part, part) {
3346	mmc_queue_suspend(&part_md->queue);
3347	}
3348	}
3349	return 0;
3350	}
3351
3352	static void mmc_blk_shutdown(struct mmc_card *card)
3353	{
3354	_mmc_blk_suspend(card);
3355	}
3356
3357	#ifdef CONFIG_PM_SLEEP
3358	static int mmc_blk_suspend(struct device *dev)
3359	{
3360	struct mmc_card *card = mmc_dev_to_card(dev);
3361
3362	return _mmc_blk_suspend(card);
3363	}
3364
3365	static int mmc_blk_resume(struct device *dev)
3366	{
3367	struct mmc_blk_data *part_md;
3368	struct mmc_blk_data *md = dev_get_drvdata(dev);
3369
3370	if (md) {
3371	/*
3372	* Resume involves the card going into idle state,
3373	* so current partition is always the main one.
3374	*/
3375	md->part_curr = md->part_type;
3376	mmc_queue_resume(&md->queue);
3377	list_for_each_entry(part_md, &md->part, part) {
3378	mmc_queue_resume(&part_md->queue);
3379	}
3380	}
3381	return 0;
3382	}
3383	#endif
3384
3385	static SIMPLE_DEV_PM_OPS(mmc_blk_pm_ops, mmc_blk_suspend, mmc_blk_resume);
3386
3387	static struct mmc_driver mmc_driver = {
3388	.drv = {
3389	.name = "mmcblk",
3390	.pm = &mmc_blk_pm_ops,
3391	},
3392	.probe = mmc_blk_probe,
3393	.remove = mmc_blk_remove,
3394	.shutdown = mmc_blk_shutdown,
3395	};
3396
3397	static int __init mmc_blk_init(void)
3398	{
3399	int res;
3400
3401	res = bus_register(bus: &mmc_rpmb_bus_type);
3402	if (res < 0) {
3403	pr_err("mmcblk: could not register RPMB bus type\n");
3404	return res;
3405	}
3406	res = alloc_chrdev_region(&mmc_rpmb_devt, 0, MAX_DEVICES, "rpmb");
3407	if (res < 0) {
3408	pr_err("mmcblk: failed to allocate rpmb chrdev region\n");
3409	goto out_bus_unreg;
3410	}
3411
3412	if (perdev_minors != CONFIG_MMC_BLOCK_MINORS)
3413	pr_info("mmcblk: using %d minors per device\n", perdev_minors);
3414
3415	max_devices = min(MAX_DEVICES, (1 << MINORBITS) / perdev_minors);
3416
3417	res = register_blkdev(MMC_BLOCK_MAJOR, "mmc");
3418	if (res)
3419	goto out_chrdev_unreg;
3420
3421	res = mmc_register_driver(drv: &mmc_driver);
3422	if (res)
3423	goto out_blkdev_unreg;
3424
3425	return 0;
3426
3427	out_blkdev_unreg:
3428	unregister_blkdev(MMC_BLOCK_MAJOR, name: "mmc");
3429	out_chrdev_unreg:
3430	unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES);
3431	out_bus_unreg:
3432	bus_unregister(bus: &mmc_rpmb_bus_type);
3433	return res;
3434	}
3435
3436	static void __exit mmc_blk_exit(void)
3437	{
3438	mmc_unregister_driver(drv: &mmc_driver);
3439	unregister_blkdev(MMC_BLOCK_MAJOR, name: "mmc");
3440	unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES);
3441	bus_unregister(bus: &mmc_rpmb_bus_type);
3442	}
3443
3444	module_init(mmc_blk_init);
3445	module_exit(mmc_blk_exit);
3446
3447	MODULE_LICENSE("GPL");
3448	MODULE_DESCRIPTION("Multimedia Card (MMC) block device driver");
3449