1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* NVM Express device driver
4	* Copyright (c) 2011-2014, Intel Corporation.
5	*/
6
7	#include <linux/acpi.h>
8	#include <linux/async.h>
9	#include <linux/blkdev.h>
10	#include <linux/blk-mq-dma.h>
11	#include <linux/blk-integrity.h>
12	#include <linux/dmi.h>
13	#include <linux/init.h>
14	#include <linux/interrupt.h>
15	#include <linux/io.h>
16	#include <linux/kstrtox.h>
17	#include <linux/memremap.h>
18	#include <linux/mm.h>
19	#include <linux/module.h>
20	#include <linux/mutex.h>
21	#include <linux/nodemask.h>
22	#include <linux/once.h>
23	#include <linux/pci.h>
24	#include <linux/suspend.h>
25	#include <linux/t10-pi.h>
26	#include <linux/types.h>
27	#include <linux/io-64-nonatomic-lo-hi.h>
28	#include <linux/io-64-nonatomic-hi-lo.h>
29	#include <linux/sed-opal.h>
30
31	#include "trace.h"
32	#include "nvme.h"
33
34	#define SQ_SIZE(q) ((q)->q_depth << (q)->sqes)
35	#define CQ_SIZE(q) ((q)->q_depth * sizeof(struct nvme_completion))
36
37	/* Optimisation for I/Os between 4k and 128k */
38	#define NVME_SMALL_POOL_SIZE 256
39
40	/*
41	* Arbitrary upper bound.
42	*/
43	#define NVME_MAX_BYTES SZ_8M
44	#define NVME_MAX_NR_DESCRIPTORS 5
45
46	/*
47	* For data SGLs we support a single descriptors worth of SGL entries.
48	* For PRPs, segments don't matter at all.
49	*/
50	#define NVME_MAX_SEGS \
51	(NVME_CTRL_PAGE_SIZE / sizeof(struct nvme_sgl_desc))
52
53	/*
54	* For metadata SGLs, only the small descriptor is supported, and the first
55	* entry is the segment descriptor, which for the data pointer sits in the SQE.
56	*/
57	#define NVME_MAX_META_SEGS \
58	((NVME_SMALL_POOL_SIZE / sizeof(struct nvme_sgl_desc)) - 1)
59
60	/*
61	* The last entry is used to link to the next descriptor.
62	*/
63	#define PRPS_PER_PAGE \
64	(((NVME_CTRL_PAGE_SIZE / sizeof(__le64))) - 1)
65
66	/*
67	* I/O could be non-aligned both at the beginning and end.
68	*/
69	#define MAX_PRP_RANGE \
70	(NVME_MAX_BYTES + 2 * (NVME_CTRL_PAGE_SIZE - 1))
71
72	static_assert(MAX_PRP_RANGE / NVME_CTRL_PAGE_SIZE <=
73	(1 /* prp1 */ + NVME_MAX_NR_DESCRIPTORS * PRPS_PER_PAGE));
74
75	static int use_threaded_interrupts;
76	module_param(use_threaded_interrupts, int, 0444);
77
78	static bool use_cmb_sqes = true;
79	module_param(use_cmb_sqes, bool, 0444);
80	MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
81
82	static unsigned int max_host_mem_size_mb = 128;
83	module_param(max_host_mem_size_mb, uint, 0444);
84	MODULE_PARM_DESC(max_host_mem_size_mb,
85	"Maximum Host Memory Buffer (HMB) size per controller (in MiB)");
86
87	static unsigned int sgl_threshold = SZ_32K;
88	module_param(sgl_threshold, uint, 0644);
89	MODULE_PARM_DESC(sgl_threshold,
90	"Use SGLs when average request segment size is larger or equal to "
91	"this size. Use 0 to disable SGLs.");
92
93	#define NVME_PCI_MIN_QUEUE_SIZE 2
94	#define NVME_PCI_MAX_QUEUE_SIZE 4095
95	static int io_queue_depth_set(const char *val, const struct kernel_param *kp);
96	static const struct kernel_param_ops io_queue_depth_ops = {
97	.set = io_queue_depth_set,
98	.get = param_get_uint,
99	};
100
101	static unsigned int io_queue_depth = 1024;
102	module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644);
103	MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2 and < 4096");
104
105	static int io_queue_count_set(const char *val, const struct kernel_param *kp)
106	{
107	unsigned int n;
108	int ret;
109
110	ret = kstrtouint(s: val, base: 10, res: &n);
111	if (ret != 0 || n > blk_mq_num_possible_queues(max_queues: 0))
112	return -EINVAL;
113	return param_set_uint(val, kp);
114	}
115
116	static const struct kernel_param_ops io_queue_count_ops = {
117	.set = io_queue_count_set,
118	.get = param_get_uint,
119	};
120
121	static unsigned int write_queues;
122	module_param_cb(write_queues, &io_queue_count_ops, &write_queues, 0644);
123	MODULE_PARM_DESC(write_queues,
124	"Number of queues to use for writes. If not set, reads and writes "
125	"will share a queue set.");
126
127	static unsigned int poll_queues;
128	module_param_cb(poll_queues, &io_queue_count_ops, &poll_queues, 0644);
129	MODULE_PARM_DESC(poll_queues, "Number of queues to use for polled IO.");
130
131	static bool noacpi;
132	module_param(noacpi, bool, 0444);
133	MODULE_PARM_DESC(noacpi, "disable acpi bios quirks");
134
135	struct nvme_dev;
136	struct nvme_queue;
137
138	static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
139	static void nvme_delete_io_queues(struct nvme_dev *dev);
140	static void nvme_update_attrs(struct nvme_dev *dev);
141
142	struct nvme_descriptor_pools {
143	struct dma_pool *large;
144	struct dma_pool *small;
145	};
146
147	/*
148	* Represents an NVM Express device. Each nvme_dev is a PCI function.
149	*/
150	struct nvme_dev {
151	struct nvme_queue *queues;
152	struct blk_mq_tag_set tagset;
153	struct blk_mq_tag_set admin_tagset;
154	u32 __iomem *dbs;
155	struct device *dev;
156	unsigned online_queues;
157	unsigned max_qid;
158	unsigned io_queues[HCTX_MAX_TYPES];
159	unsigned int num_vecs;
160	u32 q_depth;
161	int io_sqes;
162	u32 db_stride;
163	void __iomem *bar;
164	unsigned long bar_mapped_size;
165	struct mutex shutdown_lock;
166	bool subsystem;
167	u64 cmb_size;
168	bool cmb_use_sqes;
169	u32 cmbsz;
170	u32 cmbloc;
171	struct nvme_ctrl ctrl;
172	u32 last_ps;
173	bool hmb;
174	struct sg_table *hmb_sgt;
175	mempool_t *dmavec_mempool;
176
177	/* shadow doorbell buffer support: */
178	__le32 *dbbuf_dbs;
179	dma_addr_t dbbuf_dbs_dma_addr;
180	__le32 *dbbuf_eis;
181	dma_addr_t dbbuf_eis_dma_addr;
182
183	/* host memory buffer support: */
184	u64 host_mem_size;
185	u32 nr_host_mem_descs;
186	u32 host_mem_descs_size;
187	dma_addr_t host_mem_descs_dma;
188	struct nvme_host_mem_buf_desc *host_mem_descs;
189	void **host_mem_desc_bufs;
190	unsigned int nr_allocated_queues;
191	unsigned int nr_write_queues;
192	unsigned int nr_poll_queues;
193	struct nvme_descriptor_pools descriptor_pools[];
194	};
195
196	static int io_queue_depth_set(const char *val, const struct kernel_param *kp)
197	{
198	return param_set_uint_minmax(val, kp, NVME_PCI_MIN_QUEUE_SIZE,
199	NVME_PCI_MAX_QUEUE_SIZE);
200	}
201
202	static inline unsigned int sq_idx(unsigned int qid, u32 stride)
203	{
204	return qid * 2 * stride;
205	}
206
207	static inline unsigned int cq_idx(unsigned int qid, u32 stride)
208	{
209	return (qid * 2 + 1) * stride;
210	}
211
212	static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
213	{
214	return container_of(ctrl, struct nvme_dev, ctrl);
215	}
216
217	/*
218	* An NVM Express queue. Each device has at least two (one for admin
219	* commands and one for I/O commands).
220	*/
221	struct nvme_queue {
222	struct nvme_dev *dev;
223	struct nvme_descriptor_pools descriptor_pools;
224	spinlock_t sq_lock;
225	void *sq_cmds;
226	/* only used for poll queues: */
227	spinlock_t cq_poll_lock ____cacheline_aligned_in_smp;
228	struct nvme_completion *cqes;
229	dma_addr_t sq_dma_addr;
230	dma_addr_t cq_dma_addr;
231	u32 __iomem *q_db;
232	u32 q_depth;
233	u16 cq_vector;
234	u16 sq_tail;
235	u16 last_sq_tail;
236	u16 cq_head;
237	u16 qid;
238	u8 cq_phase;
239	u8 sqes;
240	unsigned long flags;
241	#define NVMEQ_ENABLED 0
242	#define NVMEQ_SQ_CMB 1
243	#define NVMEQ_DELETE_ERROR 2
244	#define NVMEQ_POLLED 3
245	__le32 *dbbuf_sq_db;
246	__le32 *dbbuf_cq_db;
247	__le32 *dbbuf_sq_ei;
248	__le32 *dbbuf_cq_ei;
249	struct completion delete_done;
250	};
251
252	/* bits for iod->flags */
253	enum nvme_iod_flags {
254	/* this command has been aborted by the timeout handler */
255	IOD_ABORTED = 1U << 0,
256
257	/* uses the small descriptor pool */
258	IOD_SMALL_DESCRIPTOR = 1U << 1,
259
260	/* single segment dma mapping */
261	IOD_SINGLE_SEGMENT = 1U << 2,
262
263	/* Data payload contains p2p memory */
264	IOD_DATA_P2P = 1U << 3,
265
266	/* Metadata contains p2p memory */
267	IOD_META_P2P = 1U << 4,
268
269	/* Data payload contains MMIO memory */
270	IOD_DATA_MMIO = 1U << 5,
271
272	/* Metadata contains MMIO memory */
273	IOD_META_MMIO = 1U << 6,
274
275	/* Metadata using non-coalesced MPTR */
276	IOD_SINGLE_META_SEGMENT = 1U << 7,
277	};
278
279	struct nvme_dma_vec {
280	dma_addr_t addr;
281	unsigned int len;
282	};
283
284	/*
285	* The nvme_iod describes the data in an I/O.
286	*/
287	struct nvme_iod {
288	struct nvme_request req;
289	struct nvme_command cmd;
290	u8 flags;
291	u8 nr_descriptors;
292
293	unsigned int total_len;
294	struct dma_iova_state dma_state;
295	void *descriptors[NVME_MAX_NR_DESCRIPTORS];
296	struct nvme_dma_vec *dma_vecs;
297	unsigned int nr_dma_vecs;
298
299	dma_addr_t meta_dma;
300	unsigned int meta_total_len;
301	struct dma_iova_state meta_dma_state;
302	struct nvme_sgl_desc *meta_descriptor;
303	};
304
305	static inline unsigned int nvme_dbbuf_size(struct nvme_dev *dev)
306	{
307	return dev->nr_allocated_queues * 8 * dev->db_stride;
308	}
309
310	static void nvme_dbbuf_dma_alloc(struct nvme_dev *dev)
311	{
312	unsigned int mem_size = nvme_dbbuf_size(dev);
313
314	if (!(dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP))
315	return;
316
317	if (dev->dbbuf_dbs) {
318	/*
319	* Clear the dbbuf memory so the driver doesn't observe stale
320	* values from the previous instantiation.
321	*/
322	memset(dev->dbbuf_dbs, 0, mem_size);
323	memset(dev->dbbuf_eis, 0, mem_size);
324	return;
325	}
326
327	dev->dbbuf_dbs = dma_alloc_coherent(dev: dev->dev, size: mem_size,
328	dma_handle: &dev->dbbuf_dbs_dma_addr,
329	GFP_KERNEL);
330	if (!dev->dbbuf_dbs)
331	goto fail;
332	dev->dbbuf_eis = dma_alloc_coherent(dev: dev->dev, size: mem_size,
333	dma_handle: &dev->dbbuf_eis_dma_addr,
334	GFP_KERNEL);
335	if (!dev->dbbuf_eis)
336	goto fail_free_dbbuf_dbs;
337	return;
338
339	fail_free_dbbuf_dbs:
340	dma_free_coherent(dev: dev->dev, size: mem_size, cpu_addr: dev->dbbuf_dbs,
341	dma_handle: dev->dbbuf_dbs_dma_addr);
342	dev->dbbuf_dbs = NULL;
343	fail:
344	dev_warn(dev->dev, "unable to allocate dma for dbbuf\n");
345	}
346
347	static void nvme_dbbuf_dma_free(struct nvme_dev *dev)
348	{
349	unsigned int mem_size = nvme_dbbuf_size(dev);
350
351	if (dev->dbbuf_dbs) {
352	dma_free_coherent(dev: dev->dev, size: mem_size,
353	cpu_addr: dev->dbbuf_dbs, dma_handle: dev->dbbuf_dbs_dma_addr);
354	dev->dbbuf_dbs = NULL;
355	}
356	if (dev->dbbuf_eis) {
357	dma_free_coherent(dev: dev->dev, size: mem_size,
358	cpu_addr: dev->dbbuf_eis, dma_handle: dev->dbbuf_eis_dma_addr);
359	dev->dbbuf_eis = NULL;
360	}
361	}
362
363	static void nvme_dbbuf_init(struct nvme_dev *dev,
364	struct nvme_queue *nvmeq, int qid)
365	{
366	if (!dev->dbbuf_dbs || !qid)
367	return;
368
369	nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, stride: dev->db_stride)];
370	nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, stride: dev->db_stride)];
371	nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, stride: dev->db_stride)];
372	nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, stride: dev->db_stride)];
373	}
374
375	static void nvme_dbbuf_free(struct nvme_queue *nvmeq)
376	{
377	if (!nvmeq->qid)
378	return;
379
380	nvmeq->dbbuf_sq_db = NULL;
381	nvmeq->dbbuf_cq_db = NULL;
382	nvmeq->dbbuf_sq_ei = NULL;
383	nvmeq->dbbuf_cq_ei = NULL;
384	}
385
386	static void nvme_dbbuf_set(struct nvme_dev *dev)
387	{
388	struct nvme_command c = { };
389	unsigned int i;
390
391	if (!dev->dbbuf_dbs)
392	return;
393
394	c.dbbuf.opcode = nvme_admin_dbbuf;
395	c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr);
396	c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr);
397
398	if (nvme_submit_sync_cmd(q: dev->ctrl.admin_q, cmd: &c, NULL, bufflen: 0)) {
399	dev_warn(dev->ctrl.device, "unable to set dbbuf\n");
400	/* Free memory and continue on */
401	nvme_dbbuf_dma_free(dev);
402
403	for (i = 1; i <= dev->online_queues; i++)
404	nvme_dbbuf_free(nvmeq: &dev->queues[i]);
405	}
406	}
407
408	static inline int nvme_dbbuf_need_event(u16 event_idx, u16 new_idx, u16 old)
409	{
410	return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old);
411	}
412
413	/* Update dbbuf and return true if an MMIO is required */
414	static bool nvme_dbbuf_update_and_check_event(u16 value, __le32 *dbbuf_db,
415	volatile __le32 *dbbuf_ei)
416	{
417	if (dbbuf_db) {
418	u16 old_value, event_idx;
419
420	/*
421	* Ensure that the queue is written before updating
422	* the doorbell in memory
423	*/
424	wmb();
425
426	old_value = le32_to_cpu(*dbbuf_db);
427	*dbbuf_db = cpu_to_le32(value);
428
429	/*
430	* Ensure that the doorbell is updated before reading the event
431	* index from memory. The controller needs to provide similar
432	* ordering to ensure the event index is updated before reading
433	* the doorbell.
434	*/
435	mb();
436
437	event_idx = le32_to_cpu(*dbbuf_ei);
438	if (!nvme_dbbuf_need_event(event_idx, new_idx: value, old: old_value))
439	return false;
440	}
441
442	return true;
443	}
444
445	static struct nvme_descriptor_pools *
446	nvme_setup_descriptor_pools(struct nvme_dev *dev, unsigned numa_node)
447	{
448	struct nvme_descriptor_pools *pools = &dev->descriptor_pools[numa_node];
449	size_t small_align = NVME_SMALL_POOL_SIZE;
450
451	if (pools->small)
452	return pools; /* already initialized */
453
454	pools->large = dma_pool_create_node(name: "nvme descriptor page", dev: dev->dev,
455	NVME_CTRL_PAGE_SIZE, NVME_CTRL_PAGE_SIZE, boundary: 0, node: numa_node);
456	if (!pools->large)
457	return ERR_PTR(error: -ENOMEM);
458
459	if (dev->ctrl.quirks & NVME_QUIRK_DMAPOOL_ALIGN_512)
460	small_align = 512;
461
462	pools->small = dma_pool_create_node(name: "nvme descriptor small", dev: dev->dev,
463	NVME_SMALL_POOL_SIZE, align: small_align, boundary: 0, node: numa_node);
464	if (!pools->small) {
465	dma_pool_destroy(pool: pools->large);
466	pools->large = NULL;
467	return ERR_PTR(error: -ENOMEM);
468	}
469
470	return pools;
471	}
472
473	static void nvme_release_descriptor_pools(struct nvme_dev *dev)
474	{
475	unsigned i;
476
477	for (i = 0; i < nr_node_ids; i++) {
478	struct nvme_descriptor_pools *pools = &dev->descriptor_pools[i];
479
480	dma_pool_destroy(pool: pools->large);
481	dma_pool_destroy(pool: pools->small);
482	}
483	}
484
485	static int nvme_init_hctx_common(struct blk_mq_hw_ctx *hctx, void *data,
486	unsigned qid)
487	{
488	struct nvme_dev *dev = to_nvme_dev(ctrl: data);
489	struct nvme_queue *nvmeq = &dev->queues[qid];
490	struct nvme_descriptor_pools *pools;
491	struct blk_mq_tags *tags;
492
493	tags = qid ? dev->tagset.tags[qid - 1] : dev->admin_tagset.tags[0];
494	WARN_ON(tags != hctx->tags);
495	pools = nvme_setup_descriptor_pools(dev, numa_node: hctx->numa_node);
496	if (IS_ERR(ptr: pools))
497	return PTR_ERR(ptr: pools);
498
499	nvmeq->descriptor_pools = *pools;
500	hctx->driver_data = nvmeq;
501	return 0;
502	}
503
504	static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
505	unsigned int hctx_idx)
506	{
507	WARN_ON(hctx_idx != 0);
508	return nvme_init_hctx_common(hctx, data, qid: 0);
509	}
510
511	static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
512	unsigned int hctx_idx)
513	{
514	return nvme_init_hctx_common(hctx, data, qid: hctx_idx + 1);
515	}
516
517	static int nvme_pci_init_request(struct blk_mq_tag_set *set,
518	struct request *req, unsigned int hctx_idx,
519	unsigned int numa_node)
520	{
521	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
522
523	nvme_req(req)->ctrl = set->driver_data;
524	nvme_req(req)->cmd = &iod->cmd;
525	return 0;
526	}
527
528	static int queue_irq_offset(struct nvme_dev *dev)
529	{
530	/* if we have more than 1 vec, admin queue offsets us by 1 */
531	if (dev->num_vecs > 1)
532	return 1;
533
534	return 0;
535	}
536
537	static void nvme_pci_map_queues(struct blk_mq_tag_set *set)
538	{
539	struct nvme_dev *dev = to_nvme_dev(ctrl: set->driver_data);
540	int i, qoff, offset;
541
542	offset = queue_irq_offset(dev);
543	for (i = 0, qoff = 0; i < set->nr_maps; i++) {
544	struct blk_mq_queue_map *map = &set->map[i];
545
546	map->nr_queues = dev->io_queues[i];
547	if (!map->nr_queues) {
548	BUG_ON(i == HCTX_TYPE_DEFAULT);
549	continue;
550	}
551
552	/*
553	* The poll queue(s) doesn't have an IRQ (and hence IRQ
554	* affinity), so use the regular blk-mq cpu mapping
555	*/
556	map->queue_offset = qoff;
557	if (i != HCTX_TYPE_POLL && offset)
558	blk_mq_map_hw_queues(qmap: map, dev: dev->dev, offset);
559	else
560	blk_mq_map_queues(qmap: map);
561	qoff += map->nr_queues;
562	offset += map->nr_queues;
563	}
564	}
565
566	/*
567	* Write sq tail if we are asked to, or if the next command would wrap.
568	*/
569	static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq)
570	{
571	if (!write_sq) {
572	u16 next_tail = nvmeq->sq_tail + 1;
573
574	if (next_tail == nvmeq->q_depth)
575	next_tail = 0;
576	if (next_tail != nvmeq->last_sq_tail)
577	return;
578	}
579
580	if (nvme_dbbuf_update_and_check_event(value: nvmeq->sq_tail,
581	dbbuf_db: nvmeq->dbbuf_sq_db, dbbuf_ei: nvmeq->dbbuf_sq_ei))
582	writel(val: nvmeq->sq_tail, addr: nvmeq->q_db);
583	nvmeq->last_sq_tail = nvmeq->sq_tail;
584	}
585
586	static inline void nvme_sq_copy_cmd(struct nvme_queue *nvmeq,
587	struct nvme_command *cmd)
588	{
589	memcpy(nvmeq->sq_cmds + (nvmeq->sq_tail << nvmeq->sqes),
590	absolute_pointer(cmd), sizeof(*cmd));
591	if (++nvmeq->sq_tail == nvmeq->q_depth)
592	nvmeq->sq_tail = 0;
593	}
594
595	static void nvme_commit_rqs(struct blk_mq_hw_ctx *hctx)
596	{
597	struct nvme_queue *nvmeq = hctx->driver_data;
598
599	spin_lock(lock: &nvmeq->sq_lock);
600	if (nvmeq->sq_tail != nvmeq->last_sq_tail)
601	nvme_write_sq_db(nvmeq, write_sq: true);
602	spin_unlock(lock: &nvmeq->sq_lock);
603	}
604
605	enum nvme_use_sgl {
606	SGL_UNSUPPORTED,
607	SGL_SUPPORTED,
608	SGL_FORCED,
609	};
610
611	static inline bool nvme_pci_metadata_use_sgls(struct request *req)
612	{
613	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
614	struct nvme_dev *dev = nvmeq->dev;
615
616	if (!nvme_ctrl_meta_sgl_supported(ctrl: &dev->ctrl))
617	return false;
618	return req->nr_integrity_segments > 1 ||
619	nvme_req(req)->flags & NVME_REQ_USERCMD;
620	}
621
622	static inline enum nvme_use_sgl nvme_pci_use_sgls(struct nvme_dev *dev,
623	struct request *req)
624	{
625	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
626
627	if (nvmeq->qid && nvme_ctrl_sgl_supported(ctrl: &dev->ctrl)) {
628	/*
629	* When the controller is capable of using SGL, there are
630	* several conditions that we force to use it:
631	*
632	* 1. A request containing page gaps within the controller's
633	* mask can not use the PRP format.
634	*
635	* 2. User commands use SGL because that lets the device
636	* validate the requested transfer lengths.
637	*
638	* 3. Multiple integrity segments must use SGL as that's the
639	* only way to describe such a command in NVMe.
640	*/
641	if (req_phys_gap_mask(req) & (NVME_CTRL_PAGE_SIZE - 1) ||
642	nvme_req(req)->flags & NVME_REQ_USERCMD ||
643	req->nr_integrity_segments > 1)
644	return SGL_FORCED;
645	return SGL_SUPPORTED;
646	}
647
648	return SGL_UNSUPPORTED;
649	}
650
651	static unsigned int nvme_pci_avg_seg_size(struct request *req)
652	{
653	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
654	unsigned int nseg;
655
656	if (blk_rq_dma_map_coalesce(state: &iod->dma_state))
657	nseg = 1;
658	else
659	nseg = blk_rq_nr_phys_segments(rq: req);
660	return DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg);
661	}
662
663	static inline struct dma_pool *nvme_dma_pool(struct nvme_queue *nvmeq,
664	struct nvme_iod *iod)
665	{
666	if (iod->flags & IOD_SMALL_DESCRIPTOR)
667	return nvmeq->descriptor_pools.small;
668	return nvmeq->descriptor_pools.large;
669	}
670
671	static inline bool nvme_pci_cmd_use_meta_sgl(struct nvme_command *cmd)
672	{
673	return (cmd->common.flags & NVME_CMD_SGL_ALL) == NVME_CMD_SGL_METASEG;
674	}
675
676	static inline bool nvme_pci_cmd_use_sgl(struct nvme_command *cmd)
677	{
678	return cmd->common.flags &
679	(NVME_CMD_SGL_METABUF | NVME_CMD_SGL_METASEG);
680	}
681
682	static inline dma_addr_t nvme_pci_first_desc_dma_addr(struct nvme_command *cmd)
683	{
684	if (nvme_pci_cmd_use_sgl(cmd))
685	return le64_to_cpu(cmd->common.dptr.sgl.addr);
686	return le64_to_cpu(cmd->common.dptr.prp2);
687	}
688
689	static void nvme_free_descriptors(struct request *req)
690	{
691	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
692	const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1;
693	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
694	dma_addr_t dma_addr = nvme_pci_first_desc_dma_addr(cmd: &iod->cmd);
695	int i;
696
697	if (iod->nr_descriptors == 1) {
698	dma_pool_free(pool: nvme_dma_pool(nvmeq, iod), vaddr: iod->descriptors[0],
699	addr: dma_addr);
700	return;
701	}
702
703	for (i = 0; i < iod->nr_descriptors; i++) {
704	__le64 *prp_list = iod->descriptors[i];
705	dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]);
706
707	dma_pool_free(pool: nvmeq->descriptor_pools.large, vaddr: prp_list,
708	addr: dma_addr);
709	dma_addr = next_dma_addr;
710	}
711	}
712
713	static void nvme_free_prps(struct request *req, unsigned int attrs)
714	{
715	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
716	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
717	unsigned int i;
718
719	for (i = 0; i < iod->nr_dma_vecs; i++)
720	dma_unmap_phys(dev: nvmeq->dev->dev, addr: iod->dma_vecs[i].addr,
721	size: iod->dma_vecs[i].len, rq_dma_dir(req), attrs);
722	mempool_free(element: iod->dma_vecs, pool: nvmeq->dev->dmavec_mempool);
723	}
724
725	static void nvme_free_sgls(struct request *req, struct nvme_sgl_desc *sge,
726	struct nvme_sgl_desc *sg_list, unsigned int attrs)
727	{
728	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
729	enum dma_data_direction dir = rq_dma_dir(req);
730	unsigned int len = le32_to_cpu(sge->length);
731	struct device *dma_dev = nvmeq->dev->dev;
732	unsigned int i;
733
734	if (sge->type == (NVME_SGL_FMT_DATA_DESC << 4)) {
735	dma_unmap_phys(dev: dma_dev, le64_to_cpu(sge->addr), size: len, dir,
736	attrs);
737	return;
738	}
739
740	for (i = 0; i < len / sizeof(*sg_list); i++)
741	dma_unmap_phys(dev: dma_dev, le64_to_cpu(sg_list[i].addr),
742	le32_to_cpu(sg_list[i].length), dir, attrs);
743	}
744
745	static void nvme_unmap_metadata(struct request *req)
746	{
747	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
748	enum pci_p2pdma_map_type map = PCI_P2PDMA_MAP_NONE;
749	enum dma_data_direction dir = rq_dma_dir(req);
750	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
751	struct device *dma_dev = nvmeq->dev->dev;
752	struct nvme_sgl_desc *sge = iod->meta_descriptor;
753	unsigned int attrs = 0;
754
755	if (iod->flags & IOD_SINGLE_META_SEGMENT) {
756	dma_unmap_page(dma_dev, iod->meta_dma,
757	rq_integrity_vec(req).bv_len,
758	rq_dma_dir(req));
759	return;
760	}
761
762	if (iod->flags & IOD_META_P2P)
763	map = PCI_P2PDMA_MAP_BUS_ADDR;
764	else if (iod->flags & IOD_META_MMIO) {
765	map = PCI_P2PDMA_MAP_THRU_HOST_BRIDGE;
766	attrs |= DMA_ATTR_MMIO;
767	}
768
769	if (!blk_rq_dma_unmap(req, dma_dev, state: &iod->meta_dma_state,
770	mapped_len: iod->meta_total_len, map)) {
771	if (nvme_pci_cmd_use_meta_sgl(cmd: &iod->cmd))
772	nvme_free_sgls(req, sge, sg_list: &sge[1], attrs);
773	else
774	dma_unmap_phys(dev: dma_dev, addr: iod->meta_dma,
775	size: iod->meta_total_len, dir, attrs);
776	}
777
778	if (iod->meta_descriptor)
779	dma_pool_free(pool: nvmeq->descriptor_pools.small,
780	vaddr: iod->meta_descriptor, addr: iod->meta_dma);
781	}
782
783	static void nvme_unmap_data(struct request *req)
784	{
785	enum pci_p2pdma_map_type map = PCI_P2PDMA_MAP_NONE;
786	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
787	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
788	struct device *dma_dev = nvmeq->dev->dev;
789	unsigned int attrs = 0;
790
791	if (iod->flags & IOD_SINGLE_SEGMENT) {
792	static_assert(offsetof(union nvme_data_ptr, prp1) ==
793	offsetof(union nvme_data_ptr, sgl.addr));
794	dma_unmap_page(dma_dev, le64_to_cpu(iod->cmd.common.dptr.prp1),
795	iod->total_len, rq_dma_dir(req));
796	return;
797	}
798
799	if (iod->flags & IOD_DATA_P2P)
800	map = PCI_P2PDMA_MAP_BUS_ADDR;
801	else if (iod->flags & IOD_DATA_MMIO) {
802	map = PCI_P2PDMA_MAP_THRU_HOST_BRIDGE;
803	attrs |= DMA_ATTR_MMIO;
804	}
805
806	if (!blk_rq_dma_unmap(req, dma_dev, state: &iod->dma_state, mapped_len: iod->total_len,
807	map)) {
808	if (nvme_pci_cmd_use_sgl(cmd: &iod->cmd))
809	nvme_free_sgls(req, sge: &iod->cmd.common.dptr.sgl,
810	sg_list: iod->descriptors[0], attrs);
811	else
812	nvme_free_prps(req, attrs);
813	}
814
815	if (iod->nr_descriptors)
816	nvme_free_descriptors(req);
817	}
818
819	static bool nvme_pci_prp_save_mapping(struct request *req,
820	struct device *dma_dev,
821	struct blk_dma_iter *iter)
822	{
823	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
824
825	if (dma_use_iova(state: &iod->dma_state) || !dma_need_unmap(dev: dma_dev))
826	return true;
827
828	if (!iod->nr_dma_vecs) {
829	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
830
831	iod->dma_vecs = mempool_alloc(nvmeq->dev->dmavec_mempool,
832	GFP_ATOMIC);
833	if (!iod->dma_vecs) {
834	iter->status = BLK_STS_RESOURCE;
835	return false;
836	}
837	}
838
839	iod->dma_vecs[iod->nr_dma_vecs].addr = iter->addr;
840	iod->dma_vecs[iod->nr_dma_vecs].len = iter->len;
841	iod->nr_dma_vecs++;
842	return true;
843	}
844
845	static bool nvme_pci_prp_iter_next(struct request *req, struct device *dma_dev,
846	struct blk_dma_iter *iter)
847	{
848	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
849
850	if (iter->len)
851	return true;
852	if (!blk_rq_dma_map_iter_next(req, dma_dev, state: &iod->dma_state, iter))
853	return false;
854	return nvme_pci_prp_save_mapping(req, dma_dev, iter);
855	}
856
857	static blk_status_t nvme_pci_setup_data_prp(struct request *req,
858	struct blk_dma_iter *iter)
859	{
860	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
861	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
862	unsigned int length = blk_rq_payload_bytes(rq: req);
863	dma_addr_t prp1_dma, prp2_dma = 0;
864	unsigned int prp_len, i;
865	__le64 *prp_list;
866
867	if (!nvme_pci_prp_save_mapping(req, dma_dev: nvmeq->dev->dev, iter))
868	return iter->status;
869
870	/*
871	* PRP1 always points to the start of the DMA transfers.
872	*
873	* This is the only PRP (except for the list entries) that could be
874	* non-aligned.
875	*/
876	prp1_dma = iter->addr;
877	prp_len = min(length, NVME_CTRL_PAGE_SIZE -
878	(iter->addr & (NVME_CTRL_PAGE_SIZE - 1)));
879	iod->total_len += prp_len;
880	iter->addr += prp_len;
881	iter->len -= prp_len;
882	length -= prp_len;
883	if (!length)
884	goto done;
885
886	if (!nvme_pci_prp_iter_next(req, dma_dev: nvmeq->dev->dev, iter)) {
887	if (WARN_ON_ONCE(!iter->status))
888	goto bad_sgl;
889	goto done;
890	}
891
892	/*
893	* PRP2 is usually a list, but can point to data if all data to be
894	* transferred fits into PRP1 + PRP2:
895	*/
896	if (length <= NVME_CTRL_PAGE_SIZE) {
897	prp2_dma = iter->addr;
898	iod->total_len += length;
899	goto done;
900	}
901
902	if (DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE) <=
903	NVME_SMALL_POOL_SIZE / sizeof(__le64))
904	iod->flags |= IOD_SMALL_DESCRIPTOR;
905
906	prp_list = dma_pool_alloc(pool: nvme_dma_pool(nvmeq, iod), GFP_ATOMIC,
907	handle: &prp2_dma);
908	if (!prp_list) {
909	iter->status = BLK_STS_RESOURCE;
910	goto done;
911	}
912	iod->descriptors[iod->nr_descriptors++] = prp_list;
913
914	i = 0;
915	for (;;) {
916	prp_list[i++] = cpu_to_le64(iter->addr);
917	prp_len = min(length, NVME_CTRL_PAGE_SIZE);
918	if (WARN_ON_ONCE(iter->len < prp_len))
919	goto bad_sgl;
920
921	iod->total_len += prp_len;
922	iter->addr += prp_len;
923	iter->len -= prp_len;
924	length -= prp_len;
925	if (!length)
926	break;
927
928	if (!nvme_pci_prp_iter_next(req, dma_dev: nvmeq->dev->dev, iter)) {
929	if (WARN_ON_ONCE(!iter->status))
930	goto bad_sgl;
931	goto done;
932	}
933
934	/*
935	* If we've filled the entire descriptor, allocate a new that is
936	* pointed to be the last entry in the previous PRP list. To
937	* accommodate for that move the last actual entry to the new
938	* descriptor.
939	*/
940	if (i == NVME_CTRL_PAGE_SIZE >> 3) {
941	__le64 *old_prp_list = prp_list;
942	dma_addr_t prp_list_dma;
943
944	prp_list = dma_pool_alloc(pool: nvmeq->descriptor_pools.large,
945	GFP_ATOMIC, handle: &prp_list_dma);
946	if (!prp_list) {
947	iter->status = BLK_STS_RESOURCE;
948	goto done;
949	}
950	iod->descriptors[iod->nr_descriptors++] = prp_list;
951
952	prp_list[0] = old_prp_list[i - 1];
953	old_prp_list[i - 1] = cpu_to_le64(prp_list_dma);
954	i = 1;
955	}
956	}
957
958	done:
959	/*
960	* nvme_unmap_data uses the DPT field in the SQE to tear down the
961	* mapping, so initialize it even for failures.
962	*/
963	iod->cmd.common.dptr.prp1 = cpu_to_le64(prp1_dma);
964	iod->cmd.common.dptr.prp2 = cpu_to_le64(prp2_dma);
965	if (unlikely(iter->status))
966	nvme_unmap_data(req);
967	return iter->status;
968
969	bad_sgl:
970	dev_err_once(nvmeq->dev->dev,
971	"Incorrectly formed request for payload:%d nents:%d\n",
972	blk_rq_payload_bytes(req), blk_rq_nr_phys_segments(req));
973	return BLK_STS_IOERR;
974	}
975
976	static void nvme_pci_sgl_set_data(struct nvme_sgl_desc *sge,
977	struct blk_dma_iter *iter)
978	{
979	sge->addr = cpu_to_le64(iter->addr);
980	sge->length = cpu_to_le32(iter->len);
981	sge->type = NVME_SGL_FMT_DATA_DESC << 4;
982	}
983
984	static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge,
985	dma_addr_t dma_addr, int entries)
986	{
987	sge->addr = cpu_to_le64(dma_addr);
988	sge->length = cpu_to_le32(entries * sizeof(*sge));
989	sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4;
990	}
991
992	static blk_status_t nvme_pci_setup_data_sgl(struct request *req,
993	struct blk_dma_iter *iter)
994	{
995	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
996	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
997	unsigned int entries = blk_rq_nr_phys_segments(rq: req);
998	struct nvme_sgl_desc *sg_list;
999	dma_addr_t sgl_dma;
1000	unsigned int mapped = 0;
1001
1002	/* set the transfer type as SGL */
1003	iod->cmd.common.flags = NVME_CMD_SGL_METABUF;
1004
1005	if (entries == 1 || blk_rq_dma_map_coalesce(state: &iod->dma_state)) {
1006	nvme_pci_sgl_set_data(sge: &iod->cmd.common.dptr.sgl, iter);
1007	iod->total_len += iter->len;
1008	return BLK_STS_OK;
1009	}
1010
1011	if (entries <= NVME_SMALL_POOL_SIZE / sizeof(*sg_list))
1012	iod->flags |= IOD_SMALL_DESCRIPTOR;
1013
1014	sg_list = dma_pool_alloc(pool: nvme_dma_pool(nvmeq, iod), GFP_ATOMIC,
1015	handle: &sgl_dma);
1016	if (!sg_list)
1017	return BLK_STS_RESOURCE;
1018	iod->descriptors[iod->nr_descriptors++] = sg_list;
1019
1020	do {
1021	if (WARN_ON_ONCE(mapped == entries)) {
1022	iter->status = BLK_STS_IOERR;
1023	break;
1024	}
1025	nvme_pci_sgl_set_data(sge: &sg_list[mapped++], iter);
1026	iod->total_len += iter->len;
1027	} while (blk_rq_dma_map_iter_next(req, dma_dev: nvmeq->dev->dev, state: &iod->dma_state,
1028	iter));
1029
1030	nvme_pci_sgl_set_seg(sge: &iod->cmd.common.dptr.sgl, dma_addr: sgl_dma, entries: mapped);
1031	if (unlikely(iter->status))
1032	nvme_unmap_data(req);
1033	return iter->status;
1034	}
1035
1036	static blk_status_t nvme_pci_setup_data_simple(struct request *req,
1037	enum nvme_use_sgl use_sgl)
1038	{
1039	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
1040	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1041	struct bio_vec bv = req_bvec(rq: req);
1042	unsigned int prp1_offset = bv.bv_offset & (NVME_CTRL_PAGE_SIZE - 1);
1043	bool prp_possible = prp1_offset + bv.bv_len <= NVME_CTRL_PAGE_SIZE * 2;
1044	dma_addr_t dma_addr;
1045
1046	if (!use_sgl && !prp_possible)
1047	return BLK_STS_AGAIN;
1048	if (is_pci_p2pdma_page(page: bv.bv_page))
1049	return BLK_STS_AGAIN;
1050
1051	dma_addr = dma_map_bvec(nvmeq->dev->dev, &bv, rq_dma_dir(req), 0);
1052	if (dma_mapping_error(dev: nvmeq->dev->dev, dma_addr))
1053	return BLK_STS_RESOURCE;
1054	iod->total_len = bv.bv_len;
1055	iod->flags |= IOD_SINGLE_SEGMENT;
1056
1057	if (use_sgl == SGL_FORCED || !prp_possible) {
1058	iod->cmd.common.flags = NVME_CMD_SGL_METABUF;
1059	iod->cmd.common.dptr.sgl.addr = cpu_to_le64(dma_addr);
1060	iod->cmd.common.dptr.sgl.length = cpu_to_le32(bv.bv_len);
1061	iod->cmd.common.dptr.sgl.type = NVME_SGL_FMT_DATA_DESC << 4;
1062	} else {
1063	unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - prp1_offset;
1064
1065	iod->cmd.common.dptr.prp1 = cpu_to_le64(dma_addr);
1066	iod->cmd.common.dptr.prp2 = 0;
1067	if (bv.bv_len > first_prp_len)
1068	iod->cmd.common.dptr.prp2 =
1069	cpu_to_le64(dma_addr + first_prp_len);
1070	}
1071
1072	return BLK_STS_OK;
1073	}
1074
1075	static blk_status_t nvme_map_data(struct request *req)
1076	{
1077	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
1078	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1079	struct nvme_dev *dev = nvmeq->dev;
1080	enum nvme_use_sgl use_sgl = nvme_pci_use_sgls(dev, req);
1081	struct blk_dma_iter iter;
1082	blk_status_t ret;
1083
1084	/*
1085	* Try to skip the DMA iterator for single segment requests, as that
1086	* significantly improves performances for small I/O sizes.
1087	*/
1088	if (blk_rq_nr_phys_segments(rq: req) == 1) {
1089	ret = nvme_pci_setup_data_simple(req, use_sgl);
1090	if (ret != BLK_STS_AGAIN)
1091	return ret;
1092	}
1093
1094	if (!blk_rq_dma_map_iter_start(req, dma_dev: dev->dev, state: &iod->dma_state, iter: &iter))
1095	return iter.status;
1096
1097	switch (iter.p2pdma.map) {
1098	case PCI_P2PDMA_MAP_BUS_ADDR:
1099	iod->flags |= IOD_DATA_P2P;
1100	break;
1101	case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
1102	iod->flags |= IOD_DATA_MMIO;
1103	break;
1104	case PCI_P2PDMA_MAP_NONE:
1105	break;
1106	default:
1107	return BLK_STS_RESOURCE;
1108	}
1109
1110	if (use_sgl == SGL_FORCED ||
1111	(use_sgl == SGL_SUPPORTED &&
1112	(sgl_threshold && nvme_pci_avg_seg_size(req) >= sgl_threshold)))
1113	return nvme_pci_setup_data_sgl(req, iter: &iter);
1114	return nvme_pci_setup_data_prp(req, iter: &iter);
1115	}
1116
1117	static blk_status_t nvme_pci_setup_meta_iter(struct request *req)
1118	{
1119	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1120	unsigned int entries = req->nr_integrity_segments;
1121	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
1122	struct nvme_dev *dev = nvmeq->dev;
1123	struct nvme_sgl_desc *sg_list;
1124	struct blk_dma_iter iter;
1125	dma_addr_t sgl_dma;
1126	int i = 0;
1127
1128	if (!blk_rq_integrity_dma_map_iter_start(req, dma_dev: dev->dev,
1129	state: &iod->meta_dma_state, iter: &iter))
1130	return iter.status;
1131
1132	switch (iter.p2pdma.map) {
1133	case PCI_P2PDMA_MAP_BUS_ADDR:
1134	iod->flags |= IOD_META_P2P;
1135	break;
1136	case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
1137	iod->flags |= IOD_META_MMIO;
1138	break;
1139	case PCI_P2PDMA_MAP_NONE:
1140	break;
1141	default:
1142	return BLK_STS_RESOURCE;
1143	}
1144
1145	if (blk_rq_dma_map_coalesce(state: &iod->meta_dma_state))
1146	entries = 1;
1147
1148	/*
1149	* The NVMe MPTR descriptor has an implicit length that the host and
1150	* device must agree on to avoid data/memory corruption. We trust the
1151	* kernel allocated correctly based on the format's parameters, so use
1152	* the more efficient MPTR to avoid extra dma pool allocations for the
1153	* SGL indirection.
1154	*
1155	* But for user commands, we don't necessarily know what they do, so
1156	* the driver can't validate the metadata buffer size. The SGL
1157	* descriptor provides an explicit length, so we're relying on that
1158	* mechanism to catch any misunderstandings between the application and
1159	* device.
1160	*
1161	* P2P DMA also needs to use the blk_dma_iter method, so mptr setup
1162	* leverages this routine when that happens.
1163	*/
1164	if (!nvme_ctrl_meta_sgl_supported(ctrl: &dev->ctrl) ||
1165	(entries == 1 && !(nvme_req(req)->flags & NVME_REQ_USERCMD))) {
1166	iod->cmd.common.metadata = cpu_to_le64(iter.addr);
1167	iod->meta_total_len = iter.len;
1168	iod->meta_dma = iter.addr;
1169	iod->meta_descriptor = NULL;
1170	return BLK_STS_OK;
1171	}
1172
1173	sg_list = dma_pool_alloc(pool: nvmeq->descriptor_pools.small, GFP_ATOMIC,
1174	handle: &sgl_dma);
1175	if (!sg_list)
1176	return BLK_STS_RESOURCE;
1177
1178	iod->meta_descriptor = sg_list;
1179	iod->meta_dma = sgl_dma;
1180	iod->cmd.common.flags = NVME_CMD_SGL_METASEG;
1181	iod->cmd.common.metadata = cpu_to_le64(sgl_dma);
1182	if (entries == 1) {
1183	iod->meta_total_len = iter.len;
1184	nvme_pci_sgl_set_data(sge: sg_list, iter: &iter);
1185	return BLK_STS_OK;
1186	}
1187
1188	sgl_dma += sizeof(*sg_list);
1189	do {
1190	nvme_pci_sgl_set_data(sge: &sg_list[++i], iter: &iter);
1191	iod->meta_total_len += iter.len;
1192	} while (blk_rq_integrity_dma_map_iter_next(req, dma_dev: dev->dev, iter: &iter));
1193
1194	nvme_pci_sgl_set_seg(sge: sg_list, dma_addr: sgl_dma, entries: i);
1195	if (unlikely(iter.status))
1196	nvme_unmap_metadata(req);
1197	return iter.status;
1198	}
1199
1200	static blk_status_t nvme_pci_setup_meta_mptr(struct request *req)
1201	{
1202	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
1203	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1204	struct bio_vec bv = rq_integrity_vec(rq: req);
1205
1206	if (is_pci_p2pdma_page(page: bv.bv_page))
1207	return nvme_pci_setup_meta_iter(req);
1208
1209	iod->meta_dma = dma_map_bvec(nvmeq->dev->dev, &bv, rq_dma_dir(req), 0);
1210	if (dma_mapping_error(dev: nvmeq->dev->dev, dma_addr: iod->meta_dma))
1211	return BLK_STS_IOERR;
1212	iod->cmd.common.metadata = cpu_to_le64(iod->meta_dma);
1213	iod->flags |= IOD_SINGLE_META_SEGMENT;
1214	return BLK_STS_OK;
1215	}
1216
1217	static blk_status_t nvme_map_metadata(struct request *req)
1218	{
1219	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
1220
1221	if ((iod->cmd.common.flags & NVME_CMD_SGL_METABUF) &&
1222	nvme_pci_metadata_use_sgls(req))
1223	return nvme_pci_setup_meta_iter(req);
1224	return nvme_pci_setup_meta_mptr(req);
1225	}
1226
1227	static blk_status_t nvme_prep_rq(struct request *req)
1228	{
1229	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
1230	blk_status_t ret;
1231
1232	iod->flags = 0;
1233	iod->nr_descriptors = 0;
1234	iod->total_len = 0;
1235	iod->meta_total_len = 0;
1236	iod->nr_dma_vecs = 0;
1237
1238	ret = nvme_setup_cmd(ns: req->q->queuedata, req);
1239	if (ret)
1240	return ret;
1241
1242	if (blk_rq_nr_phys_segments(rq: req)) {
1243	ret = nvme_map_data(req);
1244	if (ret)
1245	goto out_free_cmd;
1246	}
1247
1248	if (blk_integrity_rq(rq: req)) {
1249	ret = nvme_map_metadata(req);
1250	if (ret)
1251	goto out_unmap_data;
1252	}
1253
1254	nvme_start_request(rq: req);
1255	return BLK_STS_OK;
1256	out_unmap_data:
1257	if (blk_rq_nr_phys_segments(rq: req))
1258	nvme_unmap_data(req);
1259	out_free_cmd:
1260	nvme_cleanup_cmd(req);
1261	return ret;
1262	}
1263
1264	static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
1265	const struct blk_mq_queue_data *bd)
1266	{
1267	struct nvme_queue *nvmeq = hctx->driver_data;
1268	struct nvme_dev *dev = nvmeq->dev;
1269	struct request *req = bd->rq;
1270	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
1271	blk_status_t ret;
1272
1273	/*
1274	* We should not need to do this, but we're still using this to
1275	* ensure we can drain requests on a dying queue.
1276	*/
1277	if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags)))
1278	return BLK_STS_IOERR;
1279
1280	if (unlikely(!nvme_check_ready(&dev->ctrl, req, true)))
1281	return nvme_fail_nonready_command(ctrl: &dev->ctrl, req);
1282
1283	ret = nvme_prep_rq(req);
1284	if (unlikely(ret))
1285	return ret;
1286	spin_lock(lock: &nvmeq->sq_lock);
1287	nvme_sq_copy_cmd(nvmeq, cmd: &iod->cmd);
1288	nvme_write_sq_db(nvmeq, write_sq: bd->last);
1289	spin_unlock(lock: &nvmeq->sq_lock);
1290	return BLK_STS_OK;
1291	}
1292
1293	static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist)
1294	{
1295	struct request *req;
1296
1297	if (rq_list_empty(rl: rqlist))
1298	return;
1299
1300	spin_lock(lock: &nvmeq->sq_lock);
1301	while ((req = rq_list_pop(rl: rqlist))) {
1302	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
1303
1304	nvme_sq_copy_cmd(nvmeq, cmd: &iod->cmd);
1305	}
1306	nvme_write_sq_db(nvmeq, write_sq: true);
1307	spin_unlock(lock: &nvmeq->sq_lock);
1308	}
1309
1310	static bool nvme_prep_rq_batch(struct nvme_queue *nvmeq, struct request *req)
1311	{
1312	/*
1313	* We should not need to do this, but we're still using this to
1314	* ensure we can drain requests on a dying queue.
1315	*/
1316	if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags)))
1317	return false;
1318	if (unlikely(!nvme_check_ready(&nvmeq->dev->ctrl, req, true)))
1319	return false;
1320
1321	return nvme_prep_rq(req) == BLK_STS_OK;
1322	}
1323
1324	static void nvme_queue_rqs(struct rq_list *rqlist)
1325	{
1326	struct rq_list submit_list = { };
1327	struct rq_list requeue_list = { };
1328	struct nvme_queue *nvmeq = NULL;
1329	struct request *req;
1330
1331	while ((req = rq_list_pop(rl: rqlist))) {
1332	if (nvmeq && nvmeq != req->mq_hctx->driver_data)
1333	nvme_submit_cmds(nvmeq, rqlist: &submit_list);
1334	nvmeq = req->mq_hctx->driver_data;
1335
1336	if (nvme_prep_rq_batch(nvmeq, req))
1337	rq_list_add_tail(rl: &submit_list, rq: req);
1338	else
1339	rq_list_add_tail(rl: &requeue_list, rq: req);
1340	}
1341
1342	if (nvmeq)
1343	nvme_submit_cmds(nvmeq, rqlist: &submit_list);
1344	*rqlist = requeue_list;
1345	}
1346
1347	static __always_inline void nvme_pci_unmap_rq(struct request *req)
1348	{
1349	if (blk_integrity_rq(rq: req))
1350	nvme_unmap_metadata(req);
1351	if (blk_rq_nr_phys_segments(rq: req))
1352	nvme_unmap_data(req);
1353	}
1354
1355	static void nvme_pci_complete_rq(struct request *req)
1356	{
1357	nvme_pci_unmap_rq(req);
1358	nvme_complete_rq(req);
1359	}
1360
1361	static void nvme_pci_complete_batch(struct io_comp_batch *iob)
1362	{
1363	nvme_complete_batch(iob, fn: nvme_pci_unmap_rq);
1364	}
1365
1366	/* We read the CQE phase first to check if the rest of the entry is valid */
1367	static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq)
1368	{
1369	struct nvme_completion *hcqe = &nvmeq->cqes[nvmeq->cq_head];
1370
1371	return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == nvmeq->cq_phase;
1372	}
1373
1374	static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
1375	{
1376	u16 head = nvmeq->cq_head;
1377
1378	if (nvme_dbbuf_update_and_check_event(value: head, dbbuf_db: nvmeq->dbbuf_cq_db,
1379	dbbuf_ei: nvmeq->dbbuf_cq_ei))
1380	writel(val: head, addr: nvmeq->q_db + nvmeq->dev->db_stride);
1381	}
1382
1383	static inline struct blk_mq_tags *nvme_queue_tagset(struct nvme_queue *nvmeq)
1384	{
1385	if (!nvmeq->qid)
1386	return nvmeq->dev->admin_tagset.tags[0];
1387	return nvmeq->dev->tagset.tags[nvmeq->qid - 1];
1388	}
1389
1390	static inline void nvme_handle_cqe(struct nvme_queue *nvmeq,
1391	struct io_comp_batch *iob, u16 idx)
1392	{
1393	struct nvme_completion *cqe = &nvmeq->cqes[idx];
1394	__u16 command_id = READ_ONCE(cqe->command_id);
1395	struct request *req;
1396
1397	/*
1398	* AEN requests are special as they don't time out and can
1399	* survive any kind of queue freeze and often don't respond to
1400	* aborts. We don't even bother to allocate a struct request
1401	* for them but rather special case them here.
1402	*/
1403	if (unlikely(nvme_is_aen_req(nvmeq->qid, command_id))) {
1404	nvme_complete_async_event(ctrl: &nvmeq->dev->ctrl,
1405	status: cqe->status, res: &cqe->result);
1406	return;
1407	}
1408
1409	req = nvme_find_rq(tags: nvme_queue_tagset(nvmeq), command_id);
1410	if (unlikely(!req)) {
1411	dev_warn(nvmeq->dev->ctrl.device,
1412	"invalid id %d completed on queue %d\n",
1413	command_id, le16_to_cpu(cqe->sq_id));
1414	return;
1415	}
1416
1417	trace_nvme_sq(req, sq_head: cqe->sq_head, sq_tail: nvmeq->sq_tail);
1418	if (!nvme_try_complete_req(req, status: cqe->status, result: cqe->result) &&
1419	!blk_mq_add_to_batch(req, iob,
1420	is_error: nvme_req(req)->status != NVME_SC_SUCCESS,
1421	complete: nvme_pci_complete_batch))
1422	nvme_pci_complete_rq(req);
1423	}
1424
1425	static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
1426	{
1427	u32 tmp = nvmeq->cq_head + 1;
1428
1429	if (tmp == nvmeq->q_depth) {
1430	nvmeq->cq_head = 0;
1431	nvmeq->cq_phase ^= 1;
1432	} else {
1433	nvmeq->cq_head = tmp;
1434	}
1435	}
1436
1437	static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
1438	struct io_comp_batch *iob)
1439	{
1440	bool found = false;
1441
1442	while (nvme_cqe_pending(nvmeq)) {
1443	found = true;
1444	/*
1445	* load-load control dependency between phase and the rest of
1446	* the cqe requires a full read memory barrier
1447	*/
1448	dma_rmb();
1449	nvme_handle_cqe(nvmeq, iob, idx: nvmeq->cq_head);
1450	nvme_update_cq_head(nvmeq);
1451	}
1452
1453	if (found)
1454	nvme_ring_cq_doorbell(nvmeq);
1455	return found;
1456	}
1457
1458	static irqreturn_t nvme_irq(int irq, void *data)
1459	{
1460	struct nvme_queue *nvmeq = data;
1461	DEFINE_IO_COMP_BATCH(iob);
1462
1463	if (nvme_poll_cq(nvmeq, iob: &iob)) {
1464	if (!rq_list_empty(rl: &iob.req_list))
1465	nvme_pci_complete_batch(iob: &iob);
1466	return IRQ_HANDLED;
1467	}
1468	return IRQ_NONE;
1469	}
1470
1471	static irqreturn_t nvme_irq_check(int irq, void *data)
1472	{
1473	struct nvme_queue *nvmeq = data;
1474
1475	if (nvme_cqe_pending(nvmeq))
1476	return IRQ_WAKE_THREAD;
1477	return IRQ_NONE;
1478	}
1479
1480	/*
1481	* Poll for completions for any interrupt driven queue
1482	* Can be called from any context.
1483	*/
1484	static void nvme_poll_irqdisable(struct nvme_queue *nvmeq)
1485	{
1486	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
1487
1488	WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags));
1489
1490	disable_irq(irq: pci_irq_vector(dev: pdev, nr: nvmeq->cq_vector));
1491	spin_lock(lock: &nvmeq->cq_poll_lock);
1492	nvme_poll_cq(nvmeq, NULL);
1493	spin_unlock(lock: &nvmeq->cq_poll_lock);
1494	enable_irq(irq: pci_irq_vector(dev: pdev, nr: nvmeq->cq_vector));
1495	}
1496
1497	static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
1498	{
1499	struct nvme_queue *nvmeq = hctx->driver_data;
1500	bool found;
1501
1502	if (!nvme_cqe_pending(nvmeq))
1503	return 0;
1504
1505	spin_lock(lock: &nvmeq->cq_poll_lock);
1506	found = nvme_poll_cq(nvmeq, iob);
1507	spin_unlock(lock: &nvmeq->cq_poll_lock);
1508
1509	return found;
1510	}
1511
1512	static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl)
1513	{
1514	struct nvme_dev *dev = to_nvme_dev(ctrl);
1515	struct nvme_queue *nvmeq = &dev->queues[0];
1516	struct nvme_command c = { };
1517
1518	c.common.opcode = nvme_admin_async_event;
1519	c.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
1520
1521	spin_lock(lock: &nvmeq->sq_lock);
1522	nvme_sq_copy_cmd(nvmeq, cmd: &c);
1523	nvme_write_sq_db(nvmeq, write_sq: true);
1524	spin_unlock(lock: &nvmeq->sq_lock);
1525	}
1526
1527	static int nvme_pci_subsystem_reset(struct nvme_ctrl *ctrl)
1528	{
1529	struct nvme_dev *dev = to_nvme_dev(ctrl);
1530	int ret = 0;
1531
1532	/*
1533	* Taking the shutdown_lock ensures the BAR mapping is not being
1534	* altered by reset_work. Holding this lock before the RESETTING state
1535	* change, if successful, also ensures nvme_remove won't be able to
1536	* proceed to iounmap until we're done.
1537	*/
1538	mutex_lock(&dev->shutdown_lock);
1539	if (!dev->bar_mapped_size) {
1540	ret = -ENODEV;
1541	goto unlock;
1542	}
1543
1544	if (!nvme_change_ctrl_state(ctrl, new_state: NVME_CTRL_RESETTING)) {
1545	ret = -EBUSY;
1546	goto unlock;
1547	}
1548
1549	writel(NVME_SUBSYS_RESET, addr: dev->bar + NVME_REG_NSSR);
1550
1551	if (!nvme_change_ctrl_state(ctrl, new_state: NVME_CTRL_CONNECTING) ||
1552	!nvme_change_ctrl_state(ctrl, new_state: NVME_CTRL_LIVE))
1553	goto unlock;
1554
1555	/*
1556	* Read controller status to flush the previous write and trigger a
1557	* pcie read error.
1558	*/
1559	readl(addr: dev->bar + NVME_REG_CSTS);
1560	unlock:
1561	mutex_unlock(lock: &dev->shutdown_lock);
1562	return ret;
1563	}
1564
1565	static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
1566	{
1567	struct nvme_command c = { };
1568
1569	c.delete_queue.opcode = opcode;
1570	c.delete_queue.qid = cpu_to_le16(id);
1571
1572	return nvme_submit_sync_cmd(q: dev->ctrl.admin_q, cmd: &c, NULL, bufflen: 0);
1573	}
1574
1575	static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
1576	struct nvme_queue *nvmeq, s16 vector)
1577	{
1578	struct nvme_command c = { };
1579	int flags = NVME_QUEUE_PHYS_CONTIG;
1580
1581	if (!test_bit(NVMEQ_POLLED, &nvmeq->flags))
1582	flags |= NVME_CQ_IRQ_ENABLED;
1583
1584	/*
1585	* Note: we (ab)use the fact that the prp fields survive if no data
1586	* is attached to the request.
1587	*/
1588	c.create_cq.opcode = nvme_admin_create_cq;
1589	c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
1590	c.create_cq.cqid = cpu_to_le16(qid);
1591	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1592	c.create_cq.cq_flags = cpu_to_le16(flags);
1593	c.create_cq.irq_vector = cpu_to_le16(vector);
1594
1595	return nvme_submit_sync_cmd(q: dev->ctrl.admin_q, cmd: &c, NULL, bufflen: 0);
1596	}
1597
1598	static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
1599	struct nvme_queue *nvmeq)
1600	{
1601	struct nvme_ctrl *ctrl = &dev->ctrl;
1602	struct nvme_command c = { };
1603	int flags = NVME_QUEUE_PHYS_CONTIG;
1604
1605	/*
1606	* Some drives have a bug that auto-enables WRRU if MEDIUM isn't
1607	* set. Since URGENT priority is zeroes, it makes all queues
1608	* URGENT.
1609	*/
1610	if (ctrl->quirks & NVME_QUIRK_MEDIUM_PRIO_SQ)
1611	flags |= NVME_SQ_PRIO_MEDIUM;
1612
1613	/*
1614	* Note: we (ab)use the fact that the prp fields survive if no data
1615	* is attached to the request.
1616	*/
1617	c.create_sq.opcode = nvme_admin_create_sq;
1618	c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
1619	c.create_sq.sqid = cpu_to_le16(qid);
1620	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1621	c.create_sq.sq_flags = cpu_to_le16(flags);
1622	c.create_sq.cqid = cpu_to_le16(qid);
1623
1624	return nvme_submit_sync_cmd(q: dev->ctrl.admin_q, cmd: &c, NULL, bufflen: 0);
1625	}
1626
1627	static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
1628	{
1629	return adapter_delete_queue(dev, opcode: nvme_admin_delete_cq, id: cqid);
1630	}
1631
1632	static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
1633	{
1634	return adapter_delete_queue(dev, opcode: nvme_admin_delete_sq, id: sqid);
1635	}
1636
1637	static enum rq_end_io_ret abort_endio(struct request *req, blk_status_t error)
1638	{
1639	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1640
1641	dev_warn(nvmeq->dev->ctrl.device,
1642	"Abort status: 0x%x", nvme_req(req)->status);
1643	atomic_inc(v: &nvmeq->dev->ctrl.abort_limit);
1644	blk_mq_free_request(rq: req);
1645	return RQ_END_IO_NONE;
1646	}
1647
1648	static bool nvme_should_reset(struct nvme_dev *dev, u32 csts)
1649	{
1650	/* If true, indicates loss of adapter communication, possibly by a
1651	* NVMe Subsystem reset.
1652	*/
1653	bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO);
1654
1655	/* If there is a reset/reinit ongoing, we shouldn't reset again. */
1656	switch (nvme_ctrl_state(ctrl: &dev->ctrl)) {
1657	case NVME_CTRL_RESETTING:
1658	case NVME_CTRL_CONNECTING:
1659	return false;
1660	default:
1661	break;
1662	}
1663
1664	/* We shouldn't reset unless the controller is on fatal error state
1665	* _or_ if we lost the communication with it.
1666	*/
1667	if (!(csts & NVME_CSTS_CFS) && !nssro)
1668	return false;
1669
1670	return true;
1671	}
1672
1673	static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)
1674	{
1675	/* Read a config register to help see what died. */
1676	u16 pci_status;
1677	int result;
1678
1679	result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS,
1680	val: &pci_status);
1681	if (result == PCIBIOS_SUCCESSFUL)
1682	dev_warn(dev->ctrl.device,
1683	"controller is down; will reset: CSTS=0x%x, PCI_STATUS=0x%hx\n",
1684	csts, pci_status);
1685	else
1686	dev_warn(dev->ctrl.device,
1687	"controller is down; will reset: CSTS=0x%x, PCI_STATUS read failed (%d)\n",
1688	csts, result);
1689
1690	if (csts != ~0)
1691	return;
1692
1693	dev_warn(dev->ctrl.device,
1694	"Does your device have a faulty power saving mode enabled?\n");
1695	dev_warn(dev->ctrl.device,
1696	"Try \"nvme_core.default_ps_max_latency_us=0 pcie_aspm=off pcie_port_pm=off\" and report a bug\n");
1697	}
1698
1699	static enum blk_eh_timer_return nvme_timeout(struct request *req)
1700	{
1701	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq: req);
1702	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1703	struct nvme_dev *dev = nvmeq->dev;
1704	struct request *abort_req;
1705	struct nvme_command cmd = { };
1706	struct pci_dev *pdev = to_pci_dev(dev->dev);
1707	u32 csts = readl(addr: dev->bar + NVME_REG_CSTS);
1708	u8 opcode;
1709
1710	/*
1711	* Shutdown the device immediately if we see it is disconnected. This
1712	* unblocks PCIe error handling if the nvme driver is waiting in
1713	* error_resume for a device that has been removed. We can't unbind the
1714	* driver while the driver's error callback is waiting to complete, so
1715	* we're relying on a timeout to break that deadlock if a removal
1716	* occurs while reset work is running.
1717	*/
1718	if (pci_dev_is_disconnected(dev: pdev))
1719	nvme_change_ctrl_state(ctrl: &dev->ctrl, new_state: NVME_CTRL_DELETING);
1720	if (nvme_state_terminal(ctrl: &dev->ctrl))
1721	goto disable;
1722
1723	/* If PCI error recovery process is happening, we cannot reset or
1724	* the recovery mechanism will surely fail.
1725	*/
1726	mb();
1727	if (pci_channel_offline(pdev))
1728	return BLK_EH_RESET_TIMER;
1729
1730	/*
1731	* Reset immediately if the controller is failed
1732	*/
1733	if (nvme_should_reset(dev, csts)) {
1734	nvme_warn_reset(dev, csts);
1735	goto disable;
1736	}
1737
1738	/*
1739	* Did we miss an interrupt?
1740	*/
1741	if (test_bit(NVMEQ_POLLED, &nvmeq->flags))
1742	nvme_poll(hctx: req->mq_hctx, NULL);
1743	else
1744	nvme_poll_irqdisable(nvmeq);
1745
1746	if (blk_mq_rq_state(rq: req) != MQ_RQ_IN_FLIGHT) {
1747	dev_warn(dev->ctrl.device,
1748	"I/O tag %d (%04x) QID %d timeout, completion polled\n",
1749	req->tag, nvme_cid(req), nvmeq->qid);
1750	return BLK_EH_DONE;
1751	}
1752
1753	/*
1754	* Shutdown immediately if controller times out while starting. The
1755	* reset work will see the pci device disabled when it gets the forced
1756	* cancellation error. All outstanding requests are completed on
1757	* shutdown, so we return BLK_EH_DONE.
1758	*/
1759	switch (nvme_ctrl_state(ctrl: &dev->ctrl)) {
1760	case NVME_CTRL_CONNECTING:
1761	nvme_change_ctrl_state(ctrl: &dev->ctrl, new_state: NVME_CTRL_DELETING);
1762	fallthrough;
1763	case NVME_CTRL_DELETING:
1764	dev_warn_ratelimited(dev->ctrl.device,
1765	"I/O tag %d (%04x) QID %d timeout, disable controller\n",
1766	req->tag, nvme_cid(req), nvmeq->qid);
1767	nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1768	nvme_dev_disable(dev, shutdown: true);
1769	return BLK_EH_DONE;
1770	case NVME_CTRL_RESETTING:
1771	return BLK_EH_RESET_TIMER;
1772	default:
1773	break;
1774	}
1775
1776	/*
1777	* Shutdown the controller immediately and schedule a reset if the
1778	* command was already aborted once before and still hasn't been
1779	* returned to the driver, or if this is the admin queue.
1780	*/
1781	opcode = nvme_req(req)->cmd->common.opcode;
1782	if (!nvmeq->qid || (iod->flags & IOD_ABORTED)) {
1783	dev_warn(dev->ctrl.device,
1784	"I/O tag %d (%04x) opcode %#x (%s) QID %d timeout, reset controller\n",
1785	req->tag, nvme_cid(req), opcode,
1786	nvme_opcode_str(nvmeq->qid, opcode), nvmeq->qid);
1787	nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1788	goto disable;
1789	}
1790
1791	if (atomic_dec_return(v: &dev->ctrl.abort_limit) < 0) {
1792	atomic_inc(v: &dev->ctrl.abort_limit);
1793	return BLK_EH_RESET_TIMER;
1794	}
1795	iod->flags |= IOD_ABORTED;
1796
1797	cmd.abort.opcode = nvme_admin_abort_cmd;
1798	cmd.abort.cid = nvme_cid(rq: req);
1799	cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
1800
1801	dev_warn(nvmeq->dev->ctrl.device,
1802	"I/O tag %d (%04x) opcode %#x (%s) QID %d timeout, aborting req_op:%s(%u) size:%u\n",
1803	req->tag, nvme_cid(req), opcode, nvme_get_opcode_str(opcode),
1804	nvmeq->qid, blk_op_str(req_op(req)), req_op(req),
1805	blk_rq_bytes(req));
1806
1807	abort_req = blk_mq_alloc_request(q: dev->ctrl.admin_q, opf: nvme_req_op(cmd: &cmd),
1808	flags: BLK_MQ_REQ_NOWAIT);
1809	if (IS_ERR(ptr: abort_req)) {
1810	atomic_inc(v: &dev->ctrl.abort_limit);
1811	return BLK_EH_RESET_TIMER;
1812	}
1813	nvme_init_request(req: abort_req, cmd: &cmd);
1814
1815	abort_req->end_io = abort_endio;
1816	abort_req->end_io_data = NULL;
1817	blk_execute_rq_nowait(rq: abort_req, at_head: false);
1818
1819	/*
1820	* The aborted req will be completed on receiving the abort req.
1821	* We enable the timer again. If hit twice, it'll cause a device reset,
1822	* as the device then is in a faulty state.
1823	*/
1824	return BLK_EH_RESET_TIMER;
1825
1826	disable:
1827	if (!nvme_change_ctrl_state(ctrl: &dev->ctrl, new_state: NVME_CTRL_RESETTING)) {
1828	if (nvme_state_terminal(ctrl: &dev->ctrl))
1829	nvme_dev_disable(dev, shutdown: true);
1830	return BLK_EH_DONE;
1831	}
1832
1833	nvme_dev_disable(dev, shutdown: false);
1834	if (nvme_try_sched_reset(ctrl: &dev->ctrl))
1835	nvme_unquiesce_io_queues(ctrl: &dev->ctrl);
1836	return BLK_EH_DONE;
1837	}
1838
1839	static void nvme_free_queue(struct nvme_queue *nvmeq)
1840	{
1841	dma_free_coherent(dev: nvmeq->dev->dev, CQ_SIZE(nvmeq),
1842	cpu_addr: (void *)nvmeq->cqes, dma_handle: nvmeq->cq_dma_addr);
1843	if (!nvmeq->sq_cmds)
1844	return;
1845
1846	if (test_and_clear_bit(NVMEQ_SQ_CMB, addr: &nvmeq->flags)) {
1847	pci_free_p2pmem(to_pci_dev(nvmeq->dev->dev),
1848	addr: nvmeq->sq_cmds, SQ_SIZE(nvmeq));
1849	} else {
1850	dma_free_coherent(dev: nvmeq->dev->dev, SQ_SIZE(nvmeq),
1851	cpu_addr: nvmeq->sq_cmds, dma_handle: nvmeq->sq_dma_addr);
1852	}
1853	}
1854
1855	static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1856	{
1857	int i;
1858
1859	for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) {
1860	dev->ctrl.queue_count--;
1861	nvme_free_queue(nvmeq: &dev->queues[i]);
1862	}
1863	}
1864
1865	static void nvme_suspend_queue(struct nvme_dev *dev, unsigned int qid)
1866	{
1867	struct nvme_queue *nvmeq = &dev->queues[qid];
1868
1869	if (!test_and_clear_bit(NVMEQ_ENABLED, addr: &nvmeq->flags))
1870	return;
1871
1872	/* ensure that nvme_queue_rq() sees NVMEQ_ENABLED cleared */
1873	mb();
1874
1875	nvmeq->dev->online_queues--;
1876	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
1877	nvme_quiesce_admin_queue(ctrl: &nvmeq->dev->ctrl);
1878	if (!test_and_clear_bit(NVMEQ_POLLED, addr: &nvmeq->flags))
1879	pci_free_irq(to_pci_dev(dev->dev), nr: nvmeq->cq_vector, dev_id: nvmeq);
1880	}
1881
1882	static void nvme_suspend_io_queues(struct nvme_dev *dev)
1883	{
1884	int i;
1885
1886	for (i = dev->ctrl.queue_count - 1; i > 0; i--)
1887	nvme_suspend_queue(dev, qid: i);
1888	}
1889
1890	/*
1891	* Called only on a device that has been disabled and after all other threads
1892	* that can check this device's completion queues have synced, except
1893	* nvme_poll(). This is the last chance for the driver to see a natural
1894	* completion before nvme_cancel_request() terminates all incomplete requests.
1895	*/
1896	static void nvme_reap_pending_cqes(struct nvme_dev *dev)
1897	{
1898	int i;
1899
1900	for (i = dev->ctrl.queue_count - 1; i > 0; i--) {
1901	spin_lock(lock: &dev->queues[i].cq_poll_lock);
1902	nvme_poll_cq(nvmeq: &dev->queues[i], NULL);
1903	spin_unlock(lock: &dev->queues[i].cq_poll_lock);
1904	}
1905	}
1906
1907	static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
1908	int entry_size)
1909	{
1910	int q_depth = dev->q_depth;
1911	unsigned q_size_aligned = roundup(q_depth * entry_size,
1912	NVME_CTRL_PAGE_SIZE);
1913
1914	if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1915	u64 mem_per_q = div_u64(dividend: dev->cmb_size, divisor: nr_io_queues);
1916
1917	mem_per_q = round_down(mem_per_q, NVME_CTRL_PAGE_SIZE);
1918	q_depth = div_u64(dividend: mem_per_q, divisor: entry_size);
1919
1920	/*
1921	* Ensure the reduced q_depth is above some threshold where it
1922	* would be better to map queues in system memory with the
1923	* original depth
1924	*/
1925	if (q_depth < 64)
1926	return -ENOMEM;
1927	}
1928
1929	return q_depth;
1930	}
1931
1932	static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1933	int qid)
1934	{
1935	struct pci_dev *pdev = to_pci_dev(dev->dev);
1936
1937	if (qid && dev->cmb_use_sqes && (dev->cmbsz & NVME_CMBSZ_SQS)) {
1938	nvmeq->sq_cmds = pci_alloc_p2pmem(pdev, SQ_SIZE(nvmeq));
1939	if (nvmeq->sq_cmds) {
1940	nvmeq->sq_dma_addr = pci_p2pmem_virt_to_bus(pdev,
1941	addr: nvmeq->sq_cmds);
1942	if (nvmeq->sq_dma_addr) {
1943	set_bit(NVMEQ_SQ_CMB, addr: &nvmeq->flags);
1944	return 0;
1945	}
1946
1947	pci_free_p2pmem(pdev, addr: nvmeq->sq_cmds, SQ_SIZE(nvmeq));
1948	}
1949	}
1950
1951	nvmeq->sq_cmds = dma_alloc_coherent(dev: dev->dev, SQ_SIZE(nvmeq),
1952	dma_handle: &nvmeq->sq_dma_addr, GFP_KERNEL);
1953	if (!nvmeq->sq_cmds)
1954	return -ENOMEM;
1955	return 0;
1956	}
1957
1958	static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
1959	{
1960	struct nvme_queue *nvmeq = &dev->queues[qid];
1961
1962	if (dev->ctrl.queue_count > qid)
1963	return 0;
1964
1965	nvmeq->sqes = qid ? dev->io_sqes : NVME_ADM_SQES;
1966	nvmeq->q_depth = depth;
1967	nvmeq->cqes = dma_alloc_coherent(dev: dev->dev, CQ_SIZE(nvmeq),
1968	dma_handle: &nvmeq->cq_dma_addr, GFP_KERNEL);
1969	if (!nvmeq->cqes)
1970	goto free_nvmeq;
1971
1972	if (nvme_alloc_sq_cmds(dev, nvmeq, qid))
1973	goto free_cqdma;
1974
1975	nvmeq->dev = dev;
1976	spin_lock_init(&nvmeq->sq_lock);
1977	spin_lock_init(&nvmeq->cq_poll_lock);
1978	nvmeq->cq_head = 0;
1979	nvmeq->cq_phase = 1;
1980	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1981	nvmeq->qid = qid;
1982	dev->ctrl.queue_count++;
1983
1984	return 0;
1985
1986	free_cqdma:
1987	dma_free_coherent(dev: dev->dev, CQ_SIZE(nvmeq), cpu_addr: (void *)nvmeq->cqes,
1988	dma_handle: nvmeq->cq_dma_addr);
1989	free_nvmeq:
1990	return -ENOMEM;
1991	}
1992
1993	static int queue_request_irq(struct nvme_queue *nvmeq)
1994	{
1995	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
1996	int nr = nvmeq->dev->ctrl.instance;
1997
1998	if (use_threaded_interrupts) {
1999	return pci_request_irq(dev: pdev, nr: nvmeq->cq_vector, handler: nvme_irq_check,
2000	thread_fn: nvme_irq, dev_id: nvmeq, fmt: "nvme%dq%d", nr, nvmeq->qid);
2001	} else {
2002	return pci_request_irq(dev: pdev, nr: nvmeq->cq_vector, handler: nvme_irq,
2003	NULL, dev_id: nvmeq, fmt: "nvme%dq%d", nr, nvmeq->qid);
2004	}
2005	}
2006
2007	static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
2008	{
2009	struct nvme_dev *dev = nvmeq->dev;
2010
2011	nvmeq->sq_tail = 0;
2012	nvmeq->last_sq_tail = 0;
2013	nvmeq->cq_head = 0;
2014	nvmeq->cq_phase = 1;
2015	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
2016	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq));
2017	nvme_dbbuf_init(dev, nvmeq, qid);
2018	dev->online_queues++;
2019	wmb(); /* ensure the first interrupt sees the initialization */
2020	}
2021
2022	/*
2023	* Try getting shutdown_lock while setting up IO queues.
2024	*/
2025	static int nvme_setup_io_queues_trylock(struct nvme_dev *dev)
2026	{
2027	/*
2028	* Give up if the lock is being held by nvme_dev_disable.
2029	*/
2030	if (!mutex_trylock(&dev->shutdown_lock))
2031	return -ENODEV;
2032
2033	/*
2034	* Controller is in wrong state, fail early.
2035	*/
2036	if (nvme_ctrl_state(ctrl: &dev->ctrl) != NVME_CTRL_CONNECTING) {
2037	mutex_unlock(lock: &dev->shutdown_lock);
2038	return -ENODEV;
2039	}
2040
2041	return 0;
2042	}
2043
2044	static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled)
2045	{
2046	struct nvme_dev *dev = nvmeq->dev;
2047	int result;
2048	u16 vector = 0;
2049
2050	clear_bit(NVMEQ_DELETE_ERROR, addr: &nvmeq->flags);
2051
2052	/*
2053	* A queue's vector matches the queue identifier unless the controller
2054	* has only one vector available.
2055	*/
2056	if (!polled)
2057	vector = dev->num_vecs == 1 ? 0 : qid;
2058	else
2059	set_bit(NVMEQ_POLLED, addr: &nvmeq->flags);
2060
2061	result = adapter_alloc_cq(dev, qid, nvmeq, vector);
2062	if (result)
2063	return result;
2064
2065	result = adapter_alloc_sq(dev, qid, nvmeq);
2066	if (result < 0)
2067	return result;
2068	if (result)
2069	goto release_cq;
2070
2071	nvmeq->cq_vector = vector;
2072
2073	result = nvme_setup_io_queues_trylock(dev);
2074	if (result)
2075	return result;
2076	nvme_init_queue(nvmeq, qid);
2077	if (!polled) {
2078	result = queue_request_irq(nvmeq);
2079	if (result < 0)
2080	goto release_sq;
2081	}
2082
2083	set_bit(NVMEQ_ENABLED, addr: &nvmeq->flags);
2084	mutex_unlock(lock: &dev->shutdown_lock);
2085	return result;
2086
2087	release_sq:
2088	dev->online_queues--;
2089	mutex_unlock(lock: &dev->shutdown_lock);
2090	adapter_delete_sq(dev, sqid: qid);
2091	release_cq:
2092	adapter_delete_cq(dev, cqid: qid);
2093	return result;
2094	}
2095
2096	static const struct blk_mq_ops nvme_mq_admin_ops = {
2097	.queue_rq = nvme_queue_rq,
2098	.complete = nvme_pci_complete_rq,
2099	.init_hctx = nvme_admin_init_hctx,
2100	.init_request = nvme_pci_init_request,
2101	.timeout = nvme_timeout,
2102	};
2103
2104	static const struct blk_mq_ops nvme_mq_ops = {
2105	.queue_rq = nvme_queue_rq,
2106	.queue_rqs = nvme_queue_rqs,
2107	.complete = nvme_pci_complete_rq,
2108	.commit_rqs = nvme_commit_rqs,
2109	.init_hctx = nvme_init_hctx,
2110	.init_request = nvme_pci_init_request,
2111	.map_queues = nvme_pci_map_queues,
2112	.timeout = nvme_timeout,
2113	.poll = nvme_poll,
2114	};
2115
2116	static void nvme_dev_remove_admin(struct nvme_dev *dev)
2117	{
2118	if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
2119	/*
2120	* If the controller was reset during removal, it's possible
2121	* user requests may be waiting on a stopped queue. Start the
2122	* queue to flush these to completion.
2123	*/
2124	nvme_unquiesce_admin_queue(ctrl: &dev->ctrl);
2125	nvme_remove_admin_tag_set(ctrl: &dev->ctrl);
2126	}
2127	}
2128
2129	static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
2130	{
2131	return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride);
2132	}
2133
2134	static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size)
2135	{
2136	struct pci_dev *pdev = to_pci_dev(dev->dev);
2137
2138	if (size <= dev->bar_mapped_size)
2139	return 0;
2140	if (size > pci_resource_len(pdev, 0))
2141	return -ENOMEM;
2142	if (dev->bar)
2143	iounmap(addr: dev->bar);
2144	dev->bar = ioremap(pci_resource_start(pdev, 0), size);
2145	if (!dev->bar) {
2146	dev->bar_mapped_size = 0;
2147	return -ENOMEM;
2148	}
2149	dev->bar_mapped_size = size;
2150	dev->dbs = dev->bar + NVME_REG_DBS;
2151
2152	return 0;
2153	}
2154
2155	static int nvme_pci_configure_admin_queue(struct nvme_dev *dev)
2156	{
2157	int result;
2158	u32 aqa;
2159	struct nvme_queue *nvmeq;
2160
2161	result = nvme_remap_bar(dev, size: db_bar_size(dev, nr_io_queues: 0));
2162	if (result < 0)
2163	return result;
2164
2165	dev->subsystem = readl(addr: dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ?
2166	NVME_CAP_NSSRC(dev->ctrl.cap) : 0;
2167
2168	if (dev->subsystem &&
2169	(readl(addr: dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
2170	writel(val: NVME_CSTS_NSSRO, addr: dev->bar + NVME_REG_CSTS);
2171
2172	/*
2173	* If the device has been passed off to us in an enabled state, just
2174	* clear the enabled bit. The spec says we should set the 'shutdown
2175	* notification bits', but doing so may cause the device to complete
2176	* commands to the admin queue ... and we don't know what memory that
2177	* might be pointing at!
2178	*/
2179	result = nvme_disable_ctrl(ctrl: &dev->ctrl, shutdown: false);
2180	if (result < 0) {
2181	struct pci_dev *pdev = to_pci_dev(dev->dev);
2182
2183	/*
2184	* The NVMe Controller Reset method did not get an expected
2185	* CSTS.RDY transition, so something with the device appears to
2186	* be stuck. Use the lower level and bigger hammer PCIe
2187	* Function Level Reset to attempt restoring the device to its
2188	* initial state, and try again.
2189	*/
2190	result = pcie_reset_flr(dev: pdev, probe: false);
2191	if (result < 0)
2192	return result;
2193
2194	pci_restore_state(dev: pdev);
2195	result = nvme_disable_ctrl(ctrl: &dev->ctrl, shutdown: false);
2196	if (result < 0)
2197	return result;
2198
2199	dev_info(dev->ctrl.device,
2200	"controller reset completed after pcie flr\n");
2201	}
2202
2203	result = nvme_alloc_queue(dev, qid: 0, NVME_AQ_DEPTH);
2204	if (result)
2205	return result;
2206
2207	dev->ctrl.numa_node = dev_to_node(dev: dev->dev);
2208
2209	nvmeq = &dev->queues[0];
2210	aqa = nvmeq->q_depth - 1;
2211	aqa |= aqa << 16;
2212
2213	writel(val: aqa, addr: dev->bar + NVME_REG_AQA);
2214	lo_hi_writeq(val: nvmeq->sq_dma_addr, addr: dev->bar + NVME_REG_ASQ);
2215	lo_hi_writeq(val: nvmeq->cq_dma_addr, addr: dev->bar + NVME_REG_ACQ);
2216
2217	result = nvme_enable_ctrl(ctrl: &dev->ctrl);
2218	if (result)
2219	return result;
2220
2221	nvmeq->cq_vector = 0;
2222	nvme_init_queue(nvmeq, qid: 0);
2223	result = queue_request_irq(nvmeq);
2224	if (result) {
2225	dev->online_queues--;
2226	return result;
2227	}
2228
2229	set_bit(NVMEQ_ENABLED, addr: &nvmeq->flags);
2230	return result;
2231	}
2232
2233	static int nvme_create_io_queues(struct nvme_dev *dev)
2234	{
2235	unsigned i, max, rw_queues;
2236	int ret = 0;
2237
2238	for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) {
2239	if (nvme_alloc_queue(dev, qid: i, depth: dev->q_depth)) {
2240	ret = -ENOMEM;
2241	break;
2242	}
2243	}
2244
2245	max = min(dev->max_qid, dev->ctrl.queue_count - 1);
2246	if (max != 1 && dev->io_queues[HCTX_TYPE_POLL]) {
2247	rw_queues = dev->io_queues[HCTX_TYPE_DEFAULT] +
2248	dev->io_queues[HCTX_TYPE_READ];
2249	} else {
2250	rw_queues = max;
2251	}
2252
2253	for (i = dev->online_queues; i <= max; i++) {
2254	bool polled = i > rw_queues;
2255
2256	ret = nvme_create_queue(nvmeq: &dev->queues[i], qid: i, polled);
2257	if (ret)
2258	break;
2259	}
2260
2261	/*
2262	* Ignore failing Create SQ/CQ commands, we can continue with less
2263	* than the desired amount of queues, and even a controller without
2264	* I/O queues can still be used to issue admin commands. This might
2265	* be useful to upgrade a buggy firmware for example.
2266	*/
2267	return ret >= 0 ? 0 : ret;
2268	}
2269
2270	static u64 nvme_cmb_size_unit(struct nvme_dev *dev)
2271	{
2272	u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK;
2273
2274	return 1ULL << (12 + 4 * szu);
2275	}
2276
2277	static u32 nvme_cmb_size(struct nvme_dev *dev)
2278	{
2279	return (dev->cmbsz >> NVME_CMBSZ_SZ_SHIFT) & NVME_CMBSZ_SZ_MASK;
2280	}
2281
2282	static void nvme_map_cmb(struct nvme_dev *dev)
2283	{
2284	u64 size, offset;
2285	resource_size_t bar_size;
2286	struct pci_dev *pdev = to_pci_dev(dev->dev);
2287	int bar;
2288
2289	if (dev->cmb_size)
2290	return;
2291
2292	if (NVME_CAP_CMBS(dev->ctrl.cap))
2293	writel(val: NVME_CMBMSC_CRE, addr: dev->bar + NVME_REG_CMBMSC);
2294
2295	dev->cmbsz = readl(addr: dev->bar + NVME_REG_CMBSZ);
2296	if (!dev->cmbsz)
2297	return;
2298	dev->cmbloc = readl(addr: dev->bar + NVME_REG_CMBLOC);
2299
2300	size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev);
2301	offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc);
2302	bar = NVME_CMB_BIR(dev->cmbloc);
2303	bar_size = pci_resource_len(pdev, bar);
2304
2305	if (offset > bar_size)
2306	return;
2307
2308	/*
2309	* Controllers may support a CMB size larger than their BAR, for
2310	* example, due to being behind a bridge. Reduce the CMB to the
2311	* reported size of the BAR
2312	*/
2313	size = min(size, bar_size - offset);
2314
2315	if (!IS_ALIGNED(size, memremap_compat_align()) ||
2316	!IS_ALIGNED(pci_resource_start(pdev, bar),
2317	memremap_compat_align()))
2318	return;
2319
2320	/*
2321	* Tell the controller about the host side address mapping the CMB,
2322	* and enable CMB decoding for the NVMe 1.4+ scheme:
2323	*/
2324	if (NVME_CAP_CMBS(dev->ctrl.cap)) {
2325	hi_lo_writeq(val: NVME_CMBMSC_CRE | NVME_CMBMSC_CMSE |
2326	(pci_bus_address(pdev, bar) + offset),
2327	addr: dev->bar + NVME_REG_CMBMSC);
2328	}
2329
2330	if (pci_p2pdma_add_resource(pdev, bar, size, offset)) {
2331	dev_warn(dev->ctrl.device,
2332	"failed to register the CMB\n");
2333	hi_lo_writeq(val: 0, addr: dev->bar + NVME_REG_CMBMSC);
2334	return;
2335	}
2336
2337	dev->cmb_size = size;
2338	dev->cmb_use_sqes = use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS);
2339
2340	if ((dev->cmbsz & (NVME_CMBSZ_WDS | NVME_CMBSZ_RDS)) ==
2341	(NVME_CMBSZ_WDS | NVME_CMBSZ_RDS))
2342	pci_p2pmem_publish(pdev, publish: true);
2343	}
2344
2345	static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)
2346	{
2347	u32 host_mem_size = dev->host_mem_size >> NVME_CTRL_PAGE_SHIFT;
2348	u64 dma_addr = dev->host_mem_descs_dma;
2349	struct nvme_command c = { };
2350	int ret;
2351
2352	c.features.opcode = nvme_admin_set_features;
2353	c.features.fid = cpu_to_le32(NVME_FEAT_HOST_MEM_BUF);
2354	c.features.dword11 = cpu_to_le32(bits);
2355	c.features.dword12 = cpu_to_le32(host_mem_size);
2356	c.features.dword13 = cpu_to_le32(lower_32_bits(dma_addr));
2357	c.features.dword14 = cpu_to_le32(upper_32_bits(dma_addr));
2358	c.features.dword15 = cpu_to_le32(dev->nr_host_mem_descs);
2359
2360	ret = nvme_submit_sync_cmd(q: dev->ctrl.admin_q, cmd: &c, NULL, bufflen: 0);
2361	if (ret) {
2362	dev_warn(dev->ctrl.device,
2363	"failed to set host mem (err %d, flags %#x).\n",
2364	ret, bits);
2365	} else
2366	dev->hmb = bits & NVME_HOST_MEM_ENABLE;
2367
2368	return ret;
2369	}
2370
2371	static void nvme_free_host_mem_multi(struct nvme_dev *dev)
2372	{
2373	int i;
2374
2375	for (i = 0; i < dev->nr_host_mem_descs; i++) {
2376	struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i];
2377	size_t size = le32_to_cpu(desc->size) * NVME_CTRL_PAGE_SIZE;
2378
2379	dma_free_attrs(dev: dev->dev, size, cpu_addr: dev->host_mem_desc_bufs[i],
2380	le64_to_cpu(desc->addr),
2381	DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
2382	}
2383
2384	kfree(objp: dev->host_mem_desc_bufs);
2385	dev->host_mem_desc_bufs = NULL;
2386	}
2387
2388	static void nvme_free_host_mem(struct nvme_dev *dev)
2389	{
2390	if (dev->hmb_sgt)
2391	dma_free_noncontiguous(dev: dev->dev, size: dev->host_mem_size,
2392	sgt: dev->hmb_sgt, dir: DMA_BIDIRECTIONAL);
2393	else
2394	nvme_free_host_mem_multi(dev);
2395
2396	dma_free_coherent(dev: dev->dev, size: dev->host_mem_descs_size,
2397	cpu_addr: dev->host_mem_descs, dma_handle: dev->host_mem_descs_dma);
2398	dev->host_mem_descs = NULL;
2399	dev->host_mem_descs_size = 0;
2400	dev->nr_host_mem_descs = 0;
2401	}
2402
2403	static int nvme_alloc_host_mem_single(struct nvme_dev *dev, u64 size)
2404	{
2405	dev->hmb_sgt = dma_alloc_noncontiguous(dev: dev->dev, size,
2406	dir: DMA_BIDIRECTIONAL, GFP_KERNEL, attrs: 0);
2407	if (!dev->hmb_sgt)
2408	return -ENOMEM;
2409
2410	dev->host_mem_descs = dma_alloc_coherent(dev: dev->dev,
2411	size: sizeof(*dev->host_mem_descs), dma_handle: &dev->host_mem_descs_dma,
2412	GFP_KERNEL);
2413	if (!dev->host_mem_descs) {
2414	dma_free_noncontiguous(dev: dev->dev, size, sgt: dev->hmb_sgt,
2415	dir: DMA_BIDIRECTIONAL);
2416	dev->hmb_sgt = NULL;
2417	return -ENOMEM;
2418	}
2419	dev->host_mem_size = size;
2420	dev->host_mem_descs_size = sizeof(*dev->host_mem_descs);
2421	dev->nr_host_mem_descs = 1;
2422
2423	dev->host_mem_descs[0].addr =
2424	cpu_to_le64(dev->hmb_sgt->sgl->dma_address);
2425	dev->host_mem_descs[0].size = cpu_to_le32(size / NVME_CTRL_PAGE_SIZE);
2426	return 0;
2427	}
2428
2429	static int nvme_alloc_host_mem_multi(struct nvme_dev *dev, u64 preferred,
2430	u32 chunk_size)
2431	{
2432	struct nvme_host_mem_buf_desc *descs;
2433	u32 max_entries, len, descs_size;
2434	dma_addr_t descs_dma;
2435	int i = 0;
2436	void **bufs;
2437	u64 size, tmp;
2438
2439	tmp = (preferred + chunk_size - 1);
2440	do_div(tmp, chunk_size);
2441	max_entries = tmp;
2442
2443	if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries)
2444	max_entries = dev->ctrl.hmmaxd;
2445
2446	descs_size = max_entries * sizeof(*descs);
2447	descs = dma_alloc_coherent(dev: dev->dev, size: descs_size, dma_handle: &descs_dma,
2448	GFP_KERNEL);
2449	if (!descs)
2450	goto out;
2451
2452	bufs = kcalloc(max_entries, sizeof(*bufs), GFP_KERNEL);
2453	if (!bufs)
2454	goto out_free_descs;
2455
2456	for (size = 0; size < preferred && i < max_entries; size += len) {
2457	dma_addr_t dma_addr;
2458
2459	len = min_t(u64, chunk_size, preferred - size);
2460	bufs[i] = dma_alloc_attrs(dev: dev->dev, size: len, dma_handle: &dma_addr, GFP_KERNEL,
2461	DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
2462	if (!bufs[i])
2463	break;
2464
2465	descs[i].addr = cpu_to_le64(dma_addr);
2466	descs[i].size = cpu_to_le32(len / NVME_CTRL_PAGE_SIZE);
2467	i++;
2468	}
2469
2470	if (!size)
2471	goto out_free_bufs;
2472
2473	dev->nr_host_mem_descs = i;
2474	dev->host_mem_size = size;
2475	dev->host_mem_descs = descs;
2476	dev->host_mem_descs_dma = descs_dma;
2477	dev->host_mem_descs_size = descs_size;
2478	dev->host_mem_desc_bufs = bufs;
2479	return 0;
2480
2481	out_free_bufs:
2482	kfree(objp: bufs);
2483	out_free_descs:
2484	dma_free_coherent(dev: dev->dev, size: descs_size, cpu_addr: descs, dma_handle: descs_dma);
2485	out:
2486	dev->host_mem_descs = NULL;
2487	return -ENOMEM;
2488	}
2489
2490	static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
2491	{
2492	unsigned long dma_merge_boundary = dma_get_merge_boundary(dev: dev->dev);
2493	u64 min_chunk = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES);
2494	u64 hmminds = max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2);
2495	u64 chunk_size;
2496
2497	/*
2498	* If there is an IOMMU that can merge pages, try a virtually
2499	* non-contiguous allocation for a single segment first.
2500	*/
2501	if (dma_merge_boundary && (PAGE_SIZE & dma_merge_boundary) == 0) {
2502	if (!nvme_alloc_host_mem_single(dev, size: preferred))
2503	return 0;
2504	}
2505
2506	/* start big and work our way down */
2507	for (chunk_size = min_chunk; chunk_size >= hmminds; chunk_size /= 2) {
2508	if (!nvme_alloc_host_mem_multi(dev, preferred, chunk_size)) {
2509	if (!min || dev->host_mem_size >= min)
2510	return 0;
2511	nvme_free_host_mem(dev);
2512	}
2513	}
2514
2515	return -ENOMEM;
2516	}
2517
2518	static int nvme_setup_host_mem(struct nvme_dev *dev)
2519	{
2520	u64 max = (u64)max_host_mem_size_mb * SZ_1M;
2521	u64 preferred = (u64)dev->ctrl.hmpre * 4096;
2522	u64 min = (u64)dev->ctrl.hmmin * 4096;
2523	u32 enable_bits = NVME_HOST_MEM_ENABLE;
2524	int ret;
2525
2526	if (!dev->ctrl.hmpre)
2527	return 0;
2528
2529	preferred = min(preferred, max);
2530	if (min > max) {
2531	dev_warn(dev->ctrl.device,
2532	"min host memory (%lld MiB) above limit (%d MiB).\n",
2533	min >> ilog2(SZ_1M), max_host_mem_size_mb);
2534	nvme_free_host_mem(dev);
2535	return 0;
2536	}
2537
2538	/*
2539	* If we already have a buffer allocated check if we can reuse it.
2540	*/
2541	if (dev->host_mem_descs) {
2542	if (dev->host_mem_size >= min)
2543	enable_bits |= NVME_HOST_MEM_RETURN;
2544	else
2545	nvme_free_host_mem(dev);
2546	}
2547
2548	if (!dev->host_mem_descs) {
2549	if (nvme_alloc_host_mem(dev, min, preferred)) {
2550	dev_warn(dev->ctrl.device,
2551	"failed to allocate host memory buffer.\n");
2552	return 0; /* controller must work without HMB */
2553	}
2554
2555	dev_info(dev->ctrl.device,
2556	"allocated %lld MiB host memory buffer (%u segment%s).\n",
2557	dev->host_mem_size >> ilog2(SZ_1M),
2558	dev->nr_host_mem_descs,
2559	str_plural(dev->nr_host_mem_descs));
2560	}
2561
2562	ret = nvme_set_host_mem(dev, bits: enable_bits);
2563	if (ret)
2564	nvme_free_host_mem(dev);
2565	return ret;
2566	}
2567
2568	static ssize_t cmb_show(struct device *dev, struct device_attribute *attr,
2569	char *buf)
2570	{
2571	struct nvme_dev *ndev = to_nvme_dev(ctrl: dev_get_drvdata(dev));
2572
2573	return sysfs_emit(buf, fmt: "cmbloc : 0x%08x\ncmbsz : 0x%08x\n",
2574	ndev->cmbloc, ndev->cmbsz);
2575	}
2576	static DEVICE_ATTR_RO(cmb);
2577
2578	static ssize_t cmbloc_show(struct device *dev, struct device_attribute *attr,
2579	char *buf)
2580	{
2581	struct nvme_dev *ndev = to_nvme_dev(ctrl: dev_get_drvdata(dev));
2582
2583	return sysfs_emit(buf, fmt: "%u\n", ndev->cmbloc);
2584	}
2585	static DEVICE_ATTR_RO(cmbloc);
2586
2587	static ssize_t cmbsz_show(struct device *dev, struct device_attribute *attr,
2588	char *buf)
2589	{
2590	struct nvme_dev *ndev = to_nvme_dev(ctrl: dev_get_drvdata(dev));
2591
2592	return sysfs_emit(buf, fmt: "%u\n", ndev->cmbsz);
2593	}
2594	static DEVICE_ATTR_RO(cmbsz);
2595
2596	static ssize_t hmb_show(struct device *dev, struct device_attribute *attr,
2597	char *buf)
2598	{
2599	struct nvme_dev *ndev = to_nvme_dev(ctrl: dev_get_drvdata(dev));
2600
2601	return sysfs_emit(buf, fmt: "%d\n", ndev->hmb);
2602	}
2603
2604	static ssize_t hmb_store(struct device *dev, struct device_attribute *attr,
2605	const char *buf, size_t count)
2606	{
2607	struct nvme_dev *ndev = to_nvme_dev(ctrl: dev_get_drvdata(dev));
2608	bool new;
2609	int ret;
2610
2611	if (kstrtobool(s: buf, res: &new) < 0)
2612	return -EINVAL;
2613
2614	if (new == ndev->hmb)
2615	return count;
2616
2617	if (new) {
2618	ret = nvme_setup_host_mem(dev: ndev);
2619	} else {
2620	ret = nvme_set_host_mem(dev: ndev, bits: 0);
2621	if (!ret)
2622	nvme_free_host_mem(dev: ndev);
2623	}
2624
2625	if (ret < 0)
2626	return ret;
2627
2628	return count;
2629	}
2630	static DEVICE_ATTR_RW(hmb);
2631
2632	static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj,
2633	struct attribute *a, int n)
2634	{
2635	struct nvme_ctrl *ctrl =
2636	dev_get_drvdata(container_of(kobj, struct device, kobj));
2637	struct nvme_dev *dev = to_nvme_dev(ctrl);
2638
2639	if (a == &dev_attr_cmb.attr ||
2640	a == &dev_attr_cmbloc.attr ||
2641	a == &dev_attr_cmbsz.attr) {
2642	if (!dev->cmbsz)
2643	return 0;
2644	}
2645	if (a == &dev_attr_hmb.attr && !ctrl->hmpre)
2646	return 0;
2647
2648	return a->mode;
2649	}
2650
2651	static struct attribute *nvme_pci_attrs[] = {
2652	&dev_attr_cmb.attr,
2653	&dev_attr_cmbloc.attr,
2654	&dev_attr_cmbsz.attr,
2655	&dev_attr_hmb.attr,
2656	NULL,
2657	};
2658
2659	static const struct attribute_group nvme_pci_dev_attrs_group = {
2660	.attrs = nvme_pci_attrs,
2661	.is_visible = nvme_pci_attrs_are_visible,
2662	};
2663
2664	static const struct attribute_group *nvme_pci_dev_attr_groups[] = {
2665	&nvme_dev_attrs_group,
2666	&nvme_pci_dev_attrs_group,
2667	NULL,
2668	};
2669
2670	static void nvme_update_attrs(struct nvme_dev *dev)
2671	{
2672	sysfs_update_group(kobj: &dev->ctrl.device->kobj, grp: &nvme_pci_dev_attrs_group);
2673	}
2674
2675	/*
2676	* nirqs is the number of interrupts available for write and read
2677	* queues. The core already reserved an interrupt for the admin queue.
2678	*/
2679	static void nvme_calc_irq_sets(struct irq_affinity *affd, unsigned int nrirqs)
2680	{
2681	struct nvme_dev *dev = affd->priv;
2682	unsigned int nr_read_queues, nr_write_queues = dev->nr_write_queues;
2683
2684	/*
2685	* If there is no interrupt available for queues, ensure that
2686	* the default queue is set to 1. The affinity set size is
2687	* also set to one, but the irq core ignores it for this case.
2688	*
2689	* If only one interrupt is available or 'write_queue' == 0, combine
2690	* write and read queues.
2691	*
2692	* If 'write_queues' > 0, ensure it leaves room for at least one read
2693	* queue.
2694	*/
2695	if (!nrirqs) {
2696	nrirqs = 1;
2697	nr_read_queues = 0;
2698	} else if (nrirqs == 1 || !nr_write_queues) {
2699	nr_read_queues = 0;
2700	} else if (nr_write_queues >= nrirqs) {
2701	nr_read_queues = 1;
2702	} else {
2703	nr_read_queues = nrirqs - nr_write_queues;
2704	}
2705
2706	dev->io_queues[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues;
2707	affd->set_size[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues;
2708	dev->io_queues[HCTX_TYPE_READ] = nr_read_queues;
2709	affd->set_size[HCTX_TYPE_READ] = nr_read_queues;
2710	affd->nr_sets = nr_read_queues ? 2 : 1;
2711	}
2712
2713	static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues)
2714	{
2715	struct pci_dev *pdev = to_pci_dev(dev->dev);
2716	struct irq_affinity affd = {
2717	.pre_vectors = 1,
2718	.calc_sets = nvme_calc_irq_sets,
2719	.priv = dev,
2720	};
2721	unsigned int irq_queues, poll_queues;
2722	unsigned int flags = PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY;
2723
2724	/*
2725	* Poll queues don't need interrupts, but we need at least one I/O queue
2726	* left over for non-polled I/O.
2727	*/
2728	poll_queues = min(dev->nr_poll_queues, nr_io_queues - 1);
2729	dev->io_queues[HCTX_TYPE_POLL] = poll_queues;
2730
2731	/*
2732	* Initialize for the single interrupt case, will be updated in
2733	* nvme_calc_irq_sets().
2734	*/
2735	dev->io_queues[HCTX_TYPE_DEFAULT] = 1;
2736	dev->io_queues[HCTX_TYPE_READ] = 0;
2737
2738	/*
2739	* We need interrupts for the admin queue and each non-polled I/O queue,
2740	* but some Apple controllers require all queues to use the first
2741	* vector.
2742	*/
2743	irq_queues = 1;
2744	if (!(dev->ctrl.quirks & NVME_QUIRK_SINGLE_VECTOR))
2745	irq_queues += (nr_io_queues - poll_queues);
2746	if (dev->ctrl.quirks & NVME_QUIRK_BROKEN_MSI)
2747	flags &= ~PCI_IRQ_MSI;
2748	return pci_alloc_irq_vectors_affinity(dev: pdev, min_vecs: 1, max_vecs: irq_queues, flags,
2749	affd: &affd);
2750	}
2751
2752	static unsigned int nvme_max_io_queues(struct nvme_dev *dev)
2753	{
2754	/*
2755	* If tags are shared with admin queue (Apple bug), then
2756	* make sure we only use one IO queue.
2757	*/
2758	if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS)
2759	return 1;
2760	return blk_mq_num_possible_queues(max_queues: 0) + dev->nr_write_queues +
2761	dev->nr_poll_queues;
2762	}
2763
2764	static int nvme_setup_io_queues(struct nvme_dev *dev)
2765	{
2766	struct nvme_queue *adminq = &dev->queues[0];
2767	struct pci_dev *pdev = to_pci_dev(dev->dev);
2768	unsigned int nr_io_queues;
2769	unsigned long size;
2770	int result;
2771
2772	/*
2773	* Sample the module parameters once at reset time so that we have
2774	* stable values to work with.
2775	*/
2776	dev->nr_write_queues = write_queues;
2777	dev->nr_poll_queues = poll_queues;
2778
2779	nr_io_queues = dev->nr_allocated_queues - 1;
2780	result = nvme_set_queue_count(ctrl: &dev->ctrl, count: &nr_io_queues);
2781	if (result < 0)
2782	return result;
2783
2784	if (nr_io_queues == 0)
2785	return 0;
2786
2787	/*
2788	* Free IRQ resources as soon as NVMEQ_ENABLED bit transitions
2789	* from set to unset. If there is a window to it is truely freed,
2790	* pci_free_irq_vectors() jumping into this window will crash.
2791	* And take lock to avoid racing with pci_free_irq_vectors() in
2792	* nvme_dev_disable() path.
2793	*/
2794	result = nvme_setup_io_queues_trylock(dev);
2795	if (result)
2796	return result;
2797	if (test_and_clear_bit(NVMEQ_ENABLED, addr: &adminq->flags))
2798	pci_free_irq(dev: pdev, nr: 0, dev_id: adminq);
2799
2800	if (dev->cmb_use_sqes) {
2801	result = nvme_cmb_qdepth(dev, nr_io_queues,
2802	entry_size: sizeof(struct nvme_command));
2803	if (result > 0) {
2804	dev->q_depth = result;
2805	dev->ctrl.sqsize = result - 1;
2806	} else {
2807	dev->cmb_use_sqes = false;
2808	}
2809	}
2810
2811	do {
2812	size = db_bar_size(dev, nr_io_queues);
2813	result = nvme_remap_bar(dev, size);
2814	if (!result)
2815	break;
2816	if (!--nr_io_queues) {
2817	result = -ENOMEM;
2818	goto out_unlock;
2819	}
2820	} while (1);
2821	adminq->q_db = dev->dbs;
2822
2823	retry:
2824	/* Deregister the admin queue's interrupt */
2825	if (test_and_clear_bit(NVMEQ_ENABLED, addr: &adminq->flags))
2826	pci_free_irq(dev: pdev, nr: 0, dev_id: adminq);
2827
2828	/*
2829	* If we enable msix early due to not intx, disable it again before
2830	* setting up the full range we need.
2831	*/
2832	pci_free_irq_vectors(dev: pdev);
2833
2834	result = nvme_setup_irqs(dev, nr_io_queues);
2835	if (result <= 0) {
2836	result = -EIO;
2837	goto out_unlock;
2838	}
2839
2840	dev->num_vecs = result;
2841	result = max(result - 1, 1);
2842	dev->max_qid = result + dev->io_queues[HCTX_TYPE_POLL];
2843
2844	/*
2845	* Should investigate if there's a performance win from allocating
2846	* more queues than interrupt vectors; it might allow the submission
2847	* path to scale better, even if the receive path is limited by the
2848	* number of interrupts.
2849	*/
2850	result = queue_request_irq(nvmeq: adminq);
2851	if (result)
2852	goto out_unlock;
2853	set_bit(NVMEQ_ENABLED, addr: &adminq->flags);
2854	mutex_unlock(lock: &dev->shutdown_lock);
2855
2856	result = nvme_create_io_queues(dev);
2857	if (result || dev->online_queues < 2)
2858	return result;
2859
2860	if (dev->online_queues - 1 < dev->max_qid) {
2861	nr_io_queues = dev->online_queues - 1;
2862	nvme_delete_io_queues(dev);
2863	result = nvme_setup_io_queues_trylock(dev);
2864	if (result)
2865	return result;
2866	nvme_suspend_io_queues(dev);
2867	goto retry;
2868	}
2869	dev_info(dev->ctrl.device, "%d/%d/%d default/read/poll queues\n",
2870	dev->io_queues[HCTX_TYPE_DEFAULT],
2871	dev->io_queues[HCTX_TYPE_READ],
2872	dev->io_queues[HCTX_TYPE_POLL]);
2873	return 0;
2874	out_unlock:
2875	mutex_unlock(lock: &dev->shutdown_lock);
2876	return result;
2877	}
2878
2879	static enum rq_end_io_ret nvme_del_queue_end(struct request *req,
2880	blk_status_t error)
2881	{
2882	struct nvme_queue *nvmeq = req->end_io_data;
2883
2884	blk_mq_free_request(rq: req);
2885	complete(&nvmeq->delete_done);
2886	return RQ_END_IO_NONE;
2887	}
2888
2889	static enum rq_end_io_ret nvme_del_cq_end(struct request *req,
2890	blk_status_t error)
2891	{
2892	struct nvme_queue *nvmeq = req->end_io_data;
2893
2894	if (error)
2895	set_bit(NVMEQ_DELETE_ERROR, addr: &nvmeq->flags);
2896
2897	return nvme_del_queue_end(req, error);
2898	}
2899
2900	static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
2901	{
2902	struct request_queue *q = nvmeq->dev->ctrl.admin_q;
2903	struct request *req;
2904	struct nvme_command cmd = { };
2905
2906	cmd.delete_queue.opcode = opcode;
2907	cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid);
2908
2909	req = blk_mq_alloc_request(q, opf: nvme_req_op(cmd: &cmd), flags: BLK_MQ_REQ_NOWAIT);
2910	if (IS_ERR(ptr: req))
2911	return PTR_ERR(ptr: req);
2912	nvme_init_request(req, cmd: &cmd);
2913
2914	if (opcode == nvme_admin_delete_cq)
2915	req->end_io = nvme_del_cq_end;
2916	else
2917	req->end_io = nvme_del_queue_end;
2918	req->end_io_data = nvmeq;
2919
2920	init_completion(x: &nvmeq->delete_done);
2921	blk_execute_rq_nowait(rq: req, at_head: false);
2922	return 0;
2923	}
2924
2925	static bool __nvme_delete_io_queues(struct nvme_dev *dev, u8 opcode)
2926	{
2927	int nr_queues = dev->online_queues - 1, sent = 0;
2928	unsigned long timeout;
2929
2930	retry:
2931	timeout = NVME_ADMIN_TIMEOUT;
2932	while (nr_queues > 0) {
2933	if (nvme_delete_queue(nvmeq: &dev->queues[nr_queues], opcode))
2934	break;
2935	nr_queues--;
2936	sent++;
2937	}
2938	while (sent) {
2939	struct nvme_queue *nvmeq = &dev->queues[nr_queues + sent];
2940
2941	timeout = wait_for_completion_io_timeout(x: &nvmeq->delete_done,
2942	timeout);
2943	if (timeout == 0)
2944	return false;
2945
2946	sent--;
2947	if (nr_queues)
2948	goto retry;
2949	}
2950	return true;
2951	}
2952
2953	static void nvme_delete_io_queues(struct nvme_dev *dev)
2954	{
2955	if (__nvme_delete_io_queues(dev, opcode: nvme_admin_delete_sq))
2956	__nvme_delete_io_queues(dev, opcode: nvme_admin_delete_cq);
2957	}
2958
2959	static unsigned int nvme_pci_nr_maps(struct nvme_dev *dev)
2960	{
2961	if (dev->io_queues[HCTX_TYPE_POLL])
2962	return 3;
2963	if (dev->io_queues[HCTX_TYPE_READ])
2964	return 2;
2965	return 1;
2966	}
2967
2968	static bool nvme_pci_update_nr_queues(struct nvme_dev *dev)
2969	{
2970	if (!dev->ctrl.tagset) {
2971	nvme_alloc_io_tag_set(ctrl: &dev->ctrl, set: &dev->tagset, ops: &nvme_mq_ops,
2972	nr_maps: nvme_pci_nr_maps(dev), cmd_size: sizeof(struct nvme_iod));
2973	return true;
2974	}
2975
2976	/* Give up if we are racing with nvme_dev_disable() */
2977	if (!mutex_trylock(&dev->shutdown_lock))
2978	return false;
2979
2980	/* Check if nvme_dev_disable() has been executed already */
2981	if (!dev->online_queues) {
2982	mutex_unlock(lock: &dev->shutdown_lock);
2983	return false;
2984	}
2985
2986	blk_mq_update_nr_hw_queues(set: &dev->tagset, nr_hw_queues: dev->online_queues - 1);
2987	/* free previously allocated queues that are no longer usable */
2988	nvme_free_queues(dev, lowest: dev->online_queues);
2989	mutex_unlock(lock: &dev->shutdown_lock);
2990	return true;
2991	}
2992
2993	static int nvme_pci_enable(struct nvme_dev *dev)
2994	{
2995	int result = -ENOMEM;
2996	struct pci_dev *pdev = to_pci_dev(dev->dev);
2997	unsigned int flags = PCI_IRQ_ALL_TYPES;
2998
2999	if (pci_enable_device_mem(dev: pdev))
3000	return result;
3001
3002	pci_set_master(dev: pdev);
3003
3004	if (readl(addr: dev->bar + NVME_REG_CSTS) == -1) {
3005	dev_dbg(dev->ctrl.device, "reading CSTS register failed\n");
3006	result = -ENODEV;
3007	goto disable;
3008	}
3009
3010	/*
3011	* Some devices and/or platforms don't advertise or work with INTx
3012	* interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll
3013	* adjust this later.
3014	*/
3015	if (dev->ctrl.quirks & NVME_QUIRK_BROKEN_MSI)
3016	flags &= ~PCI_IRQ_MSI;
3017	result = pci_alloc_irq_vectors(dev: pdev, min_vecs: 1, max_vecs: 1, flags);
3018	if (result < 0)
3019	goto disable;
3020
3021	dev->ctrl.cap = lo_hi_readq(addr: dev->bar + NVME_REG_CAP);
3022
3023	dev->q_depth = min_t(u32, NVME_CAP_MQES(dev->ctrl.cap) + 1,
3024	io_queue_depth);
3025	dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap);
3026	dev->dbs = dev->bar + 4096;
3027
3028	/*
3029	* Some Apple controllers require a non-standard SQE size.
3030	* Interestingly they also seem to ignore the CC:IOSQES register
3031	* so we don't bother updating it here.
3032	*/
3033	if (dev->ctrl.quirks & NVME_QUIRK_128_BYTES_SQES)
3034	dev->io_sqes = 7;
3035	else
3036	dev->io_sqes = NVME_NVM_IOSQES;
3037
3038	if (dev->ctrl.quirks & NVME_QUIRK_QDEPTH_ONE) {
3039	dev->q_depth = 2;
3040	} else if (pdev->vendor == PCI_VENDOR_ID_SAMSUNG &&
3041	(pdev->device == 0xa821 || pdev->device == 0xa822) &&
3042	NVME_CAP_MQES(dev->ctrl.cap) == 0) {
3043	dev->q_depth = 64;
3044	dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, "
3045	"set queue depth=%u\n", dev->q_depth);
3046	}
3047
3048	/*
3049	* Controllers with the shared tags quirk need the IO queue to be
3050	* big enough so that we get 32 tags for the admin queue
3051	*/
3052	if ((dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) &&
3053	(dev->q_depth < (NVME_AQ_DEPTH + 2))) {
3054	dev->q_depth = NVME_AQ_DEPTH + 2;
3055	dev_warn(dev->ctrl.device, "IO queue depth clamped to %d\n",
3056	dev->q_depth);
3057	}
3058	dev->ctrl.sqsize = dev->q_depth - 1; /* 0's based queue depth */
3059
3060	nvme_map_cmb(dev);
3061
3062	pci_save_state(dev: pdev);
3063
3064	result = nvme_pci_configure_admin_queue(dev);
3065	if (result)
3066	goto free_irq;
3067	return result;
3068
3069	free_irq:
3070	pci_free_irq_vectors(dev: pdev);
3071	disable:
3072	pci_disable_device(dev: pdev);
3073	return result;
3074	}
3075
3076	static void nvme_dev_unmap(struct nvme_dev *dev)
3077	{
3078	if (dev->bar)
3079	iounmap(addr: dev->bar);
3080	pci_release_mem_regions(to_pci_dev(dev->dev));
3081	}
3082
3083	static bool nvme_pci_ctrl_is_dead(struct nvme_dev *dev)
3084	{
3085	struct pci_dev *pdev = to_pci_dev(dev->dev);
3086	u32 csts;
3087
3088	if (!pci_is_enabled(pdev) || !pci_device_is_present(pdev))
3089	return true;
3090	if (pdev->error_state != pci_channel_io_normal)
3091	return true;
3092
3093	csts = readl(addr: dev->bar + NVME_REG_CSTS);
3094	return (csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY);
3095	}
3096
3097	static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
3098	{
3099	enum nvme_ctrl_state state = nvme_ctrl_state(ctrl: &dev->ctrl);
3100	struct pci_dev *pdev = to_pci_dev(dev->dev);
3101	bool dead;
3102
3103	mutex_lock(&dev->shutdown_lock);
3104	dead = nvme_pci_ctrl_is_dead(dev);
3105	if (state == NVME_CTRL_LIVE || state == NVME_CTRL_RESETTING) {
3106	if (pci_is_enabled(pdev))
3107	nvme_start_freeze(ctrl: &dev->ctrl);
3108	/*
3109	* Give the controller a chance to complete all entered requests
3110	* if doing a safe shutdown.
3111	*/
3112	if (!dead && shutdown)
3113	nvme_wait_freeze_timeout(ctrl: &dev->ctrl, NVME_IO_TIMEOUT);
3114	}
3115
3116	nvme_quiesce_io_queues(ctrl: &dev->ctrl);
3117
3118	if (!dead && dev->ctrl.queue_count > 0) {
3119	nvme_delete_io_queues(dev);
3120	nvme_disable_ctrl(ctrl: &dev->ctrl, shutdown);
3121	nvme_poll_irqdisable(nvmeq: &dev->queues[0]);
3122	}
3123	nvme_suspend_io_queues(dev);
3124	nvme_suspend_queue(dev, qid: 0);
3125	pci_free_irq_vectors(dev: pdev);
3126	if (pci_is_enabled(pdev))
3127	pci_disable_device(dev: pdev);
3128	nvme_reap_pending_cqes(dev);
3129
3130	nvme_cancel_tagset(ctrl: &dev->ctrl);
3131	nvme_cancel_admin_tagset(ctrl: &dev->ctrl);
3132
3133	/*
3134	* The driver will not be starting up queues again if shutting down so
3135	* must flush all entered requests to their failed completion to avoid
3136	* deadlocking blk-mq hot-cpu notifier.
3137	*/
3138	if (shutdown) {
3139	nvme_unquiesce_io_queues(ctrl: &dev->ctrl);
3140	if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q))
3141	nvme_unquiesce_admin_queue(ctrl: &dev->ctrl);
3142	}
3143	mutex_unlock(lock: &dev->shutdown_lock);
3144	}
3145
3146	static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown)
3147	{
3148	if (!nvme_wait_reset(ctrl: &dev->ctrl))
3149	return -EBUSY;
3150	nvme_dev_disable(dev, shutdown);
3151	return 0;
3152	}
3153
3154	static int nvme_pci_alloc_iod_mempool(struct nvme_dev *dev)
3155	{
3156	size_t alloc_size = sizeof(struct nvme_dma_vec) * NVME_MAX_SEGS;
3157
3158	dev->dmavec_mempool = mempool_create_node(1,
3159	mempool_kmalloc, mempool_kfree,
3160	(void *)alloc_size, GFP_KERNEL,
3161	dev_to_node(dev->dev));
3162	if (!dev->dmavec_mempool)
3163	return -ENOMEM;
3164	return 0;
3165	}
3166
3167	static void nvme_free_tagset(struct nvme_dev *dev)
3168	{
3169	if (dev->tagset.tags)
3170	nvme_remove_io_tag_set(ctrl: &dev->ctrl);
3171	dev->ctrl.tagset = NULL;
3172	}
3173
3174	/* pairs with nvme_pci_alloc_dev */
3175	static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
3176	{
3177	struct nvme_dev *dev = to_nvme_dev(ctrl);
3178
3179	nvme_free_tagset(dev);
3180	put_device(dev: dev->dev);
3181	kfree(objp: dev->queues);
3182	kfree(objp: dev);
3183	}
3184
3185	static void nvme_reset_work(struct work_struct *work)
3186	{
3187	struct nvme_dev *dev =
3188	container_of(work, struct nvme_dev, ctrl.reset_work);
3189	bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL);
3190	int result;
3191
3192	if (nvme_ctrl_state(ctrl: &dev->ctrl) != NVME_CTRL_RESETTING) {
3193	dev_warn(dev->ctrl.device, "ctrl state %d is not RESETTING\n",
3194	dev->ctrl.state);
3195	result = -ENODEV;
3196	goto out;
3197	}
3198
3199	/*
3200	* If we're called to reset a live controller first shut it down before
3201	* moving on.
3202	*/
3203	if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
3204	nvme_dev_disable(dev, shutdown: false);
3205	nvme_sync_queues(ctrl: &dev->ctrl);
3206
3207	mutex_lock(&dev->shutdown_lock);
3208	result = nvme_pci_enable(dev);
3209	if (result)
3210	goto out_unlock;
3211	nvme_unquiesce_admin_queue(ctrl: &dev->ctrl);
3212	mutex_unlock(lock: &dev->shutdown_lock);
3213
3214	/*
3215	* Introduce CONNECTING state from nvme-fc/rdma transports to mark the
3216	* initializing procedure here.
3217	*/
3218	if (!nvme_change_ctrl_state(ctrl: &dev->ctrl, new_state: NVME_CTRL_CONNECTING)) {
3219	dev_warn(dev->ctrl.device,
3220	"failed to mark controller CONNECTING\n");
3221	result = -EBUSY;
3222	goto out;
3223	}
3224
3225	result = nvme_init_ctrl_finish(ctrl: &dev->ctrl, was_suspended: was_suspend);
3226	if (result)
3227	goto out;
3228
3229	if (nvme_ctrl_meta_sgl_supported(ctrl: &dev->ctrl))
3230	dev->ctrl.max_integrity_segments = NVME_MAX_META_SEGS;
3231	else
3232	dev->ctrl.max_integrity_segments = 1;
3233
3234	nvme_dbbuf_dma_alloc(dev);
3235
3236	result = nvme_setup_host_mem(dev);
3237	if (result < 0)
3238	goto out;
3239
3240	nvme_update_attrs(dev);
3241
3242	result = nvme_setup_io_queues(dev);
3243	if (result)
3244	goto out;
3245
3246	/*
3247	* Freeze and update the number of I/O queues as those might have
3248	* changed. If there are no I/O queues left after this reset, keep the
3249	* controller around but remove all namespaces.
3250	*/
3251	if (dev->online_queues > 1) {
3252	nvme_dbbuf_set(dev);
3253	nvme_unquiesce_io_queues(ctrl: &dev->ctrl);
3254	nvme_wait_freeze(ctrl: &dev->ctrl);
3255	if (!nvme_pci_update_nr_queues(dev))
3256	goto out;
3257	nvme_unfreeze(ctrl: &dev->ctrl);
3258	} else {
3259	dev_warn(dev->ctrl.device, "IO queues lost\n");
3260	nvme_mark_namespaces_dead(ctrl: &dev->ctrl);
3261	nvme_unquiesce_io_queues(ctrl: &dev->ctrl);
3262	nvme_remove_namespaces(ctrl: &dev->ctrl);
3263	nvme_free_tagset(dev);
3264	}
3265
3266	/*
3267	* If only admin queue live, keep it to do further investigation or
3268	* recovery.
3269	*/
3270	if (!nvme_change_ctrl_state(ctrl: &dev->ctrl, new_state: NVME_CTRL_LIVE)) {
3271	dev_warn(dev->ctrl.device,
3272	"failed to mark controller live state\n");
3273	result = -ENODEV;
3274	goto out;
3275	}
3276
3277	nvme_start_ctrl(ctrl: &dev->ctrl);
3278	return;
3279
3280	out_unlock:
3281	mutex_unlock(lock: &dev->shutdown_lock);
3282	out:
3283	/*
3284	* Set state to deleting now to avoid blocking nvme_wait_reset(), which
3285	* may be holding this pci_dev's device lock.
3286	*/
3287	dev_warn(dev->ctrl.device, "Disabling device after reset failure: %d\n",
3288	result);
3289	nvme_change_ctrl_state(ctrl: &dev->ctrl, new_state: NVME_CTRL_DELETING);
3290	nvme_dev_disable(dev, shutdown: true);
3291	nvme_sync_queues(ctrl: &dev->ctrl);
3292	nvme_mark_namespaces_dead(ctrl: &dev->ctrl);
3293	nvme_unquiesce_io_queues(ctrl: &dev->ctrl);
3294	nvme_change_ctrl_state(ctrl: &dev->ctrl, new_state: NVME_CTRL_DEAD);
3295	}
3296
3297	static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
3298	{
3299	*val = readl(addr: to_nvme_dev(ctrl)->bar + off);
3300	return 0;
3301	}
3302
3303	static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
3304	{
3305	writel(val, addr: to_nvme_dev(ctrl)->bar + off);
3306	return 0;
3307	}
3308
3309	static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
3310	{
3311	*val = lo_hi_readq(addr: to_nvme_dev(ctrl)->bar + off);
3312	return 0;
3313	}
3314
3315	static int nvme_pci_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
3316	{
3317	struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
3318
3319	return snprintf(buf, size, fmt: "%s\n", dev_name(dev: &pdev->dev));
3320	}
3321
3322	static void nvme_pci_print_device_info(struct nvme_ctrl *ctrl)
3323	{
3324	struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
3325	struct nvme_subsystem *subsys = ctrl->subsys;
3326
3327	dev_err(ctrl->device,
3328	"VID:DID %04x:%04x model:%.*s firmware:%.*s\n",
3329	pdev->vendor, pdev->device,
3330	nvme_strlen(subsys->model, sizeof(subsys->model)),
3331	subsys->model, nvme_strlen(subsys->firmware_rev,
3332	sizeof(subsys->firmware_rev)),
3333	subsys->firmware_rev);
3334	}
3335
3336	static bool nvme_pci_supports_pci_p2pdma(struct nvme_ctrl *ctrl)
3337	{
3338	struct nvme_dev *dev = to_nvme_dev(ctrl);
3339
3340	return dma_pci_p2pdma_supported(dev: dev->dev);
3341	}
3342
3343	static unsigned long nvme_pci_get_virt_boundary(struct nvme_ctrl *ctrl,
3344	bool is_admin)
3345	{
3346	if (!nvme_ctrl_sgl_supported(ctrl) || is_admin)
3347	return NVME_CTRL_PAGE_SIZE - 1;
3348	return 0;
3349	}
3350
3351	static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
3352	.name = "pcie",
3353	.module = THIS_MODULE,
3354	.flags = NVME_F_METADATA_SUPPORTED,
3355	.dev_attr_groups = nvme_pci_dev_attr_groups,
3356	.reg_read32 = nvme_pci_reg_read32,
3357	.reg_write32 = nvme_pci_reg_write32,
3358	.reg_read64 = nvme_pci_reg_read64,
3359	.free_ctrl = nvme_pci_free_ctrl,
3360	.submit_async_event = nvme_pci_submit_async_event,
3361	.subsystem_reset = nvme_pci_subsystem_reset,
3362	.get_address = nvme_pci_get_address,
3363	.print_device_info = nvme_pci_print_device_info,
3364	.supports_pci_p2pdma = nvme_pci_supports_pci_p2pdma,
3365	.get_virt_boundary = nvme_pci_get_virt_boundary,
3366	};
3367
3368	static int nvme_dev_map(struct nvme_dev *dev)
3369	{
3370	struct pci_dev *pdev = to_pci_dev(dev->dev);
3371
3372	if (pci_request_mem_regions(pdev, name: "nvme"))
3373	return -ENODEV;
3374
3375	if (nvme_remap_bar(dev, size: NVME_REG_DBS + 4096))
3376	goto release;
3377
3378	return 0;
3379	release:
3380	pci_release_mem_regions(pdev);
3381	return -ENODEV;
3382	}
3383
3384	static unsigned long check_vendor_combination_bug(struct pci_dev *pdev)
3385	{
3386	if (pdev->vendor == 0x144d && pdev->device == 0xa802) {
3387	/*
3388	* Several Samsung devices seem to drop off the PCIe bus
3389	* randomly when APST is on and uses the deepest sleep state.
3390	* This has been observed on a Samsung "SM951 NVMe SAMSUNG
3391	* 256GB", a "PM951 NVMe SAMSUNG 512GB", and a "Samsung SSD
3392	* 950 PRO 256GB", but it seems to be restricted to two Dell
3393	* laptops.
3394	*/
3395	if (dmi_match(f: DMI_SYS_VENDOR, str: "Dell Inc.") &&
3396	(dmi_match(f: DMI_PRODUCT_NAME, str: "XPS 15 9550") ||
3397	dmi_match(f: DMI_PRODUCT_NAME, str: "Precision 5510")))
3398	return NVME_QUIRK_NO_DEEPEST_PS;
3399	} else if (pdev->vendor == 0x144d && pdev->device == 0xa804) {
3400	/*
3401	* Samsung SSD 960 EVO drops off the PCIe bus after system
3402	* suspend on a Ryzen board, ASUS PRIME B350M-A, as well as
3403	* within few minutes after bootup on a Coffee Lake board -
3404	* ASUS PRIME Z370-A
3405	*/
3406	if (dmi_match(f: DMI_BOARD_VENDOR, str: "ASUSTeK COMPUTER INC.") &&
3407	(dmi_match(f: DMI_BOARD_NAME, str: "PRIME B350M-A") ||
3408	dmi_match(f: DMI_BOARD_NAME, str: "PRIME Z370-A")))
3409	return NVME_QUIRK_NO_APST;
3410	} else if ((pdev->vendor == 0x144d && (pdev->device == 0xa801 ||
3411	pdev->device == 0xa808 || pdev->device == 0xa809)) ||
3412	(pdev->vendor == 0x1e0f && pdev->device == 0x0001)) {
3413	/*
3414	* Forcing to use host managed nvme power settings for
3415	* lowest idle power with quick resume latency on
3416	* Samsung and Toshiba SSDs based on suspend behavior
3417	* on Coffee Lake board for LENOVO C640
3418	*/
3419	if ((dmi_match(f: DMI_BOARD_VENDOR, str: "LENOVO")) &&
3420	dmi_match(f: DMI_BOARD_NAME, str: "LNVNB161216"))
3421	return NVME_QUIRK_SIMPLE_SUSPEND;
3422	} else if (pdev->vendor == 0x2646 && (pdev->device == 0x2263 ||
3423	pdev->device == 0x500f)) {
3424	/*
3425	* Exclude some Kingston NV1 and A2000 devices from
3426	* NVME_QUIRK_SIMPLE_SUSPEND. Do a full suspend to save a
3427	* lot of energy with s2idle sleep on some TUXEDO platforms.
3428	*/
3429	if (dmi_match(f: DMI_BOARD_NAME, str: "NS5X_NS7XAU") ||
3430	dmi_match(f: DMI_BOARD_NAME, str: "NS5x_7xAU") ||
3431	dmi_match(f: DMI_BOARD_NAME, str: "NS5x_7xPU") ||
3432	dmi_match(f: DMI_BOARD_NAME, str: "PH4PRX1_PH6PRX1"))
3433	return NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND;
3434	} else if (pdev->vendor == 0x144d && pdev->device == 0xa80d) {
3435	/*
3436	* Exclude Samsung 990 Evo from NVME_QUIRK_SIMPLE_SUSPEND
3437	* because of high power consumption (> 2 Watt) in s2idle
3438	* sleep. Only some boards with Intel CPU are affected.
3439	* (Note for testing: Samsung 990 Evo Plus has same PCI ID)
3440	*/
3441	if (dmi_match(f: DMI_BOARD_NAME, str: "DN50Z-140HC-YD") ||
3442	dmi_match(f: DMI_BOARD_NAME, str: "GMxPXxx") ||
3443	dmi_match(f: DMI_BOARD_NAME, str: "GXxMRXx") ||
3444	dmi_match(f: DMI_BOARD_NAME, str: "NS5X_NS7XAU") ||
3445	dmi_match(f: DMI_BOARD_NAME, str: "PH4PG31") ||
3446	dmi_match(f: DMI_BOARD_NAME, str: "PH4PRX1_PH6PRX1") ||
3447	dmi_match(f: DMI_BOARD_NAME, str: "PH6PG01_PH6PG71"))
3448	return NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND;
3449	}
3450
3451	/*
3452	* NVMe SSD drops off the PCIe bus after system idle
3453	* for 10 hours on a Lenovo N60z board.
3454	*/
3455	if (dmi_match(f: DMI_BOARD_NAME, str: "LXKT-ZXEG-N6"))
3456	return NVME_QUIRK_NO_APST;
3457
3458	return 0;
3459	}
3460
3461	static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev,
3462	const struct pci_device_id *id)
3463	{
3464	unsigned long quirks = id->driver_data;
3465	int node = dev_to_node(dev: &pdev->dev);
3466	struct nvme_dev *dev;
3467	int ret = -ENOMEM;
3468
3469	dev = kzalloc_node(struct_size(dev, descriptor_pools, nr_node_ids),
3470	GFP_KERNEL, node);
3471	if (!dev)
3472	return ERR_PTR(error: -ENOMEM);
3473	INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
3474	mutex_init(&dev->shutdown_lock);
3475
3476	dev->nr_write_queues = write_queues;
3477	dev->nr_poll_queues = poll_queues;
3478	dev->nr_allocated_queues = nvme_max_io_queues(dev) + 1;
3479	dev->queues = kcalloc_node(dev->nr_allocated_queues,
3480	sizeof(struct nvme_queue), GFP_KERNEL, node);
3481	if (!dev->queues)
3482	goto out_free_dev;
3483
3484	dev->dev = get_device(dev: &pdev->dev);
3485
3486	quirks |= check_vendor_combination_bug(pdev);
3487	if (!noacpi &&
3488	!(quirks & NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND) &&
3489	acpi_storage_d3(dev: &pdev->dev)) {
3490	/*
3491	* Some systems use a bios work around to ask for D3 on
3492	* platforms that support kernel managed suspend.
3493	*/
3494	dev_info(&pdev->dev,
3495	"platform quirk: setting simple suspend\n");
3496	quirks |= NVME_QUIRK_SIMPLE_SUSPEND;
3497	}
3498	ret = nvme_init_ctrl(ctrl: &dev->ctrl, dev: &pdev->dev, ops: &nvme_pci_ctrl_ops,
3499	quirks);
3500	if (ret)
3501	goto out_put_device;
3502
3503	if (dev->ctrl.quirks & NVME_QUIRK_DMA_ADDRESS_BITS_48)
3504	dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(48));
3505	else
3506	dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
3507	dma_set_min_align_mask(dev: &pdev->dev, NVME_CTRL_PAGE_SIZE - 1);
3508	dma_set_max_seg_size(dev: &pdev->dev, size: 0xffffffff);
3509
3510	/*
3511	* Limit the max command size to prevent iod->sg allocations going
3512	* over a single page.
3513	*/
3514	dev->ctrl.max_hw_sectors = min_t(u32,
3515	NVME_MAX_BYTES >> SECTOR_SHIFT,
3516	dma_opt_mapping_size(&pdev->dev) >> 9);
3517	dev->ctrl.max_segments = NVME_MAX_SEGS;
3518	dev->ctrl.max_integrity_segments = 1;
3519	return dev;
3520
3521	out_put_device:
3522	put_device(dev: dev->dev);
3523	kfree(objp: dev->queues);
3524	out_free_dev:
3525	kfree(objp: dev);
3526	return ERR_PTR(error: ret);
3527	}
3528
3529	static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3530	{
3531	struct nvme_dev *dev;
3532	int result = -ENOMEM;
3533
3534	dev = nvme_pci_alloc_dev(pdev, id);
3535	if (IS_ERR(ptr: dev))
3536	return PTR_ERR(ptr: dev);
3537
3538	result = nvme_add_ctrl(ctrl: &dev->ctrl);
3539	if (result)
3540	goto out_put_ctrl;
3541
3542	result = nvme_dev_map(dev);
3543	if (result)
3544	goto out_uninit_ctrl;
3545
3546	result = nvme_pci_alloc_iod_mempool(dev);
3547	if (result)
3548	goto out_dev_unmap;
3549
3550	dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));
3551
3552	result = nvme_pci_enable(dev);
3553	if (result)
3554	goto out_release_iod_mempool;
3555
3556	result = nvme_alloc_admin_tag_set(ctrl: &dev->ctrl, set: &dev->admin_tagset,
3557	ops: &nvme_mq_admin_ops, cmd_size: sizeof(struct nvme_iod));
3558	if (result)
3559	goto out_disable;
3560
3561	/*
3562	* Mark the controller as connecting before sending admin commands to
3563	* allow the timeout handler to do the right thing.
3564	*/
3565	if (!nvme_change_ctrl_state(ctrl: &dev->ctrl, new_state: NVME_CTRL_CONNECTING)) {
3566	dev_warn(dev->ctrl.device,
3567	"failed to mark controller CONNECTING\n");
3568	result = -EBUSY;
3569	goto out_disable;
3570	}
3571
3572	result = nvme_init_ctrl_finish(ctrl: &dev->ctrl, was_suspended: false);
3573	if (result)
3574	goto out_disable;
3575
3576	if (nvme_ctrl_meta_sgl_supported(ctrl: &dev->ctrl))
3577	dev->ctrl.max_integrity_segments = NVME_MAX_META_SEGS;
3578	else
3579	dev->ctrl.max_integrity_segments = 1;
3580
3581	nvme_dbbuf_dma_alloc(dev);
3582
3583	result = nvme_setup_host_mem(dev);
3584	if (result < 0)
3585	goto out_disable;
3586
3587	nvme_update_attrs(dev);
3588
3589	result = nvme_setup_io_queues(dev);
3590	if (result)
3591	goto out_disable;
3592
3593	if (dev->online_queues > 1) {
3594	nvme_alloc_io_tag_set(ctrl: &dev->ctrl, set: &dev->tagset, ops: &nvme_mq_ops,
3595	nr_maps: nvme_pci_nr_maps(dev), cmd_size: sizeof(struct nvme_iod));
3596	nvme_dbbuf_set(dev);
3597	}
3598
3599	if (!dev->ctrl.tagset)
3600	dev_warn(dev->ctrl.device, "IO queues not created\n");
3601
3602	if (!nvme_change_ctrl_state(ctrl: &dev->ctrl, new_state: NVME_CTRL_LIVE)) {
3603	dev_warn(dev->ctrl.device,
3604	"failed to mark controller live state\n");
3605	result = -ENODEV;
3606	goto out_disable;
3607	}
3608
3609	pci_set_drvdata(pdev, data: dev);
3610
3611	nvme_start_ctrl(ctrl: &dev->ctrl);
3612	nvme_put_ctrl(ctrl: &dev->ctrl);
3613	flush_work(work: &dev->ctrl.scan_work);
3614	return 0;
3615
3616	out_disable:
3617	nvme_change_ctrl_state(ctrl: &dev->ctrl, new_state: NVME_CTRL_DELETING);
3618	nvme_dev_disable(dev, shutdown: true);
3619	nvme_free_host_mem(dev);
3620	nvme_dev_remove_admin(dev);
3621	nvme_dbbuf_dma_free(dev);
3622	nvme_free_queues(dev, lowest: 0);
3623	out_release_iod_mempool:
3624	mempool_destroy(pool: dev->dmavec_mempool);
3625	out_dev_unmap:
3626	nvme_dev_unmap(dev);
3627	out_uninit_ctrl:
3628	nvme_uninit_ctrl(ctrl: &dev->ctrl);
3629	out_put_ctrl:
3630	nvme_put_ctrl(ctrl: &dev->ctrl);
3631	dev_err_probe(dev: &pdev->dev, err: result, fmt: "probe failed\n");
3632	return result;
3633	}
3634
3635	static void nvme_reset_prepare(struct pci_dev *pdev)
3636	{
3637	struct nvme_dev *dev = pci_get_drvdata(pdev);
3638
3639	/*
3640	* We don't need to check the return value from waiting for the reset
3641	* state as pci_dev device lock is held, making it impossible to race
3642	* with ->remove().
3643	*/
3644	nvme_disable_prepare_reset(dev, shutdown: false);
3645	nvme_sync_queues(ctrl: &dev->ctrl);
3646	}
3647
3648	static void nvme_reset_done(struct pci_dev *pdev)
3649	{
3650	struct nvme_dev *dev = pci_get_drvdata(pdev);
3651
3652	if (!nvme_try_sched_reset(ctrl: &dev->ctrl))
3653	flush_work(work: &dev->ctrl.reset_work);
3654	}
3655
3656	static void nvme_shutdown(struct pci_dev *pdev)
3657	{
3658	struct nvme_dev *dev = pci_get_drvdata(pdev);
3659
3660	nvme_disable_prepare_reset(dev, shutdown: true);
3661	}
3662
3663	/*
3664	* The driver's remove may be called on a device in a partially initialized
3665	* state. This function must not have any dependencies on the device state in
3666	* order to proceed.
3667	*/
3668	static void nvme_remove(struct pci_dev *pdev)
3669	{
3670	struct nvme_dev *dev = pci_get_drvdata(pdev);
3671
3672	nvme_change_ctrl_state(ctrl: &dev->ctrl, new_state: NVME_CTRL_DELETING);
3673	pci_set_drvdata(pdev, NULL);
3674
3675	if (!pci_device_is_present(pdev)) {
3676	nvme_change_ctrl_state(ctrl: &dev->ctrl, new_state: NVME_CTRL_DEAD);
3677	nvme_dev_disable(dev, shutdown: true);
3678	}
3679
3680	flush_work(work: &dev->ctrl.reset_work);
3681	nvme_stop_ctrl(ctrl: &dev->ctrl);
3682	nvme_remove_namespaces(ctrl: &dev->ctrl);
3683	nvme_dev_disable(dev, shutdown: true);
3684	nvme_free_host_mem(dev);
3685	nvme_dev_remove_admin(dev);
3686	nvme_dbbuf_dma_free(dev);
3687	nvme_free_queues(dev, lowest: 0);
3688	mempool_destroy(pool: dev->dmavec_mempool);
3689	nvme_release_descriptor_pools(dev);
3690	nvme_dev_unmap(dev);
3691	nvme_uninit_ctrl(ctrl: &dev->ctrl);
3692	}
3693
3694	#ifdef CONFIG_PM_SLEEP
3695	static int nvme_get_power_state(struct nvme_ctrl *ctrl, u32 *ps)
3696	{
3697	return nvme_get_features(dev: ctrl, fid: NVME_FEAT_POWER_MGMT, dword11: 0, NULL, buflen: 0, result: ps);
3698	}
3699
3700	static int nvme_set_power_state(struct nvme_ctrl *ctrl, u32 ps)
3701	{
3702	return nvme_set_features(dev: ctrl, fid: NVME_FEAT_POWER_MGMT, dword11: ps, NULL, buflen: 0, NULL);
3703	}
3704
3705	static int nvme_resume(struct device *dev)
3706	{
3707	struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
3708	struct nvme_ctrl *ctrl = &ndev->ctrl;
3709
3710	if (ndev->last_ps == U32_MAX ||
3711	nvme_set_power_state(ctrl, ps: ndev->last_ps) != 0)
3712	goto reset;
3713	if (ctrl->hmpre && nvme_setup_host_mem(dev: ndev))
3714	goto reset;
3715
3716	return 0;
3717	reset:
3718	return nvme_try_sched_reset(ctrl);
3719	}
3720
3721	static int nvme_suspend(struct device *dev)
3722	{
3723	struct pci_dev *pdev = to_pci_dev(dev);
3724	struct nvme_dev *ndev = pci_get_drvdata(pdev);
3725	struct nvme_ctrl *ctrl = &ndev->ctrl;
3726	int ret = -EBUSY;
3727
3728	ndev->last_ps = U32_MAX;
3729
3730	/*
3731	* The platform does not remove power for a kernel managed suspend so
3732	* use host managed nvme power settings for lowest idle power if
3733	* possible. This should have quicker resume latency than a full device
3734	* shutdown. But if the firmware is involved after the suspend or the
3735	* device does not support any non-default power states, shut down the
3736	* device fully.
3737	*
3738	* If ASPM is not enabled for the device, shut down the device and allow
3739	* the PCI bus layer to put it into D3 in order to take the PCIe link
3740	* down, so as to allow the platform to achieve its minimum low-power
3741	* state (which may not be possible if the link is up).
3742	*/
3743	if (pm_suspend_via_firmware() || !ctrl->npss ||
3744	!pcie_aspm_enabled(pdev) ||
3745	(ndev->ctrl.quirks & NVME_QUIRK_SIMPLE_SUSPEND))
3746	return nvme_disable_prepare_reset(dev: ndev, shutdown: true);
3747
3748	nvme_start_freeze(ctrl);
3749	nvme_wait_freeze(ctrl);
3750	nvme_sync_queues(ctrl);
3751
3752	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
3753	goto unfreeze;
3754
3755	/*
3756	* Host memory access may not be successful in a system suspend state,
3757	* but the specification allows the controller to access memory in a
3758	* non-operational power state.
3759	*/
3760	if (ndev->hmb) {
3761	ret = nvme_set_host_mem(dev: ndev, bits: 0);
3762	if (ret < 0)
3763	goto unfreeze;
3764	}
3765
3766	ret = nvme_get_power_state(ctrl, ps: &ndev->last_ps);
3767	if (ret < 0)
3768	goto unfreeze;
3769
3770	/*
3771	* A saved state prevents pci pm from generically controlling the
3772	* device's power. If we're using protocol specific settings, we don't
3773	* want pci interfering.
3774	*/
3775	pci_save_state(dev: pdev);
3776
3777	ret = nvme_set_power_state(ctrl, ps: ctrl->npss);
3778	if (ret < 0)
3779	goto unfreeze;
3780
3781	if (ret) {
3782	/* discard the saved state */
3783	pci_load_saved_state(dev: pdev, NULL);
3784
3785	/*
3786	* Clearing npss forces a controller reset on resume. The
3787	* correct value will be rediscovered then.
3788	*/
3789	ret = nvme_disable_prepare_reset(dev: ndev, shutdown: true);
3790	ctrl->npss = 0;
3791	}
3792	unfreeze:
3793	nvme_unfreeze(ctrl);
3794	return ret;
3795	}
3796
3797	static int nvme_simple_suspend(struct device *dev)
3798	{
3799	struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
3800
3801	return nvme_disable_prepare_reset(dev: ndev, shutdown: true);
3802	}
3803
3804	static int nvme_simple_resume(struct device *dev)
3805	{
3806	struct pci_dev *pdev = to_pci_dev(dev);
3807	struct nvme_dev *ndev = pci_get_drvdata(pdev);
3808
3809	return nvme_try_sched_reset(ctrl: &ndev->ctrl);
3810	}
3811
3812	static const struct dev_pm_ops nvme_dev_pm_ops = {
3813	.suspend = nvme_suspend,
3814	.resume = nvme_resume,
3815	.freeze = nvme_simple_suspend,
3816	.thaw = nvme_simple_resume,
3817	.poweroff = nvme_simple_suspend,
3818	.restore = nvme_simple_resume,
3819	};
3820	#endif /* CONFIG_PM_SLEEP */
3821
3822	static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
3823	pci_channel_state_t state)
3824	{
3825	struct nvme_dev *dev = pci_get_drvdata(pdev);
3826
3827	/*
3828	* A frozen channel requires a reset. When detected, this method will
3829	* shutdown the controller to quiesce. The controller will be restarted
3830	* after the slot reset through driver's slot_reset callback.
3831	*/
3832	switch (state) {
3833	case pci_channel_io_normal:
3834	return PCI_ERS_RESULT_CAN_RECOVER;
3835	case pci_channel_io_frozen:
3836	dev_warn(dev->ctrl.device,
3837	"frozen state error detected, reset controller\n");
3838	if (!nvme_change_ctrl_state(ctrl: &dev->ctrl, new_state: NVME_CTRL_RESETTING)) {
3839	nvme_dev_disable(dev, shutdown: true);
3840	return PCI_ERS_RESULT_DISCONNECT;
3841	}
3842	nvme_dev_disable(dev, shutdown: false);
3843	return PCI_ERS_RESULT_NEED_RESET;
3844	case pci_channel_io_perm_failure:
3845	dev_warn(dev->ctrl.device,
3846	"failure state error detected, request disconnect\n");
3847	return PCI_ERS_RESULT_DISCONNECT;
3848	}
3849	return PCI_ERS_RESULT_NEED_RESET;
3850	}
3851
3852	static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev)
3853	{
3854	struct nvme_dev *dev = pci_get_drvdata(pdev);
3855
3856	dev_info(dev->ctrl.device, "restart after slot reset\n");
3857	pci_restore_state(dev: pdev);
3858	if (nvme_try_sched_reset(ctrl: &dev->ctrl))
3859	nvme_unquiesce_io_queues(ctrl: &dev->ctrl);
3860	return PCI_ERS_RESULT_RECOVERED;
3861	}
3862
3863	static void nvme_error_resume(struct pci_dev *pdev)
3864	{
3865	struct nvme_dev *dev = pci_get_drvdata(pdev);
3866
3867	flush_work(work: &dev->ctrl.reset_work);
3868	}
3869
3870	static const struct pci_error_handlers nvme_err_handler = {
3871	.error_detected = nvme_error_detected,
3872	.slot_reset = nvme_slot_reset,
3873	.resume = nvme_error_resume,
3874	.reset_prepare = nvme_reset_prepare,
3875	.reset_done = nvme_reset_done,
3876	};
3877
3878	static const struct pci_device_id nvme_id_table[] = {
3879	{ PCI_VDEVICE(INTEL, 0x0953), /* Intel 750/P3500/P3600/P3700 */
3880	.driver_data = NVME_QUIRK_STRIPE_SIZE |
3881	NVME_QUIRK_DEALLOCATE_ZEROES, },
3882	{ PCI_VDEVICE(INTEL, 0x0a53), /* Intel P3520 */
3883	.driver_data = NVME_QUIRK_STRIPE_SIZE |
3884	NVME_QUIRK_DEALLOCATE_ZEROES, },
3885	{ PCI_VDEVICE(INTEL, 0x0a54), /* Intel P4500/P4600 */
3886	.driver_data = NVME_QUIRK_STRIPE_SIZE |
3887	NVME_QUIRK_IGNORE_DEV_SUBNQN |
3888	NVME_QUIRK_BOGUS_NID, },
3889	{ PCI_VDEVICE(INTEL, 0x0a55), /* Dell Express Flash P4600 */
3890	.driver_data = NVME_QUIRK_STRIPE_SIZE, },
3891	{ PCI_VDEVICE(INTEL, 0xf1a5), /* Intel 600P/P3100 */
3892	.driver_data = NVME_QUIRK_NO_DEEPEST_PS |
3893	NVME_QUIRK_MEDIUM_PRIO_SQ |
3894	NVME_QUIRK_NO_TEMP_THRESH_CHANGE |
3895	NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3896	{ PCI_VDEVICE(INTEL, 0xf1a6), /* Intel 760p/Pro 7600p */
3897	.driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3898	{ PCI_VDEVICE(INTEL, 0x5845), /* Qemu emulated controller */
3899	.driver_data = NVME_QUIRK_IDENTIFY_CNS |
3900	NVME_QUIRK_DISABLE_WRITE_ZEROES |
3901	NVME_QUIRK_BOGUS_NID, },
3902	{ PCI_VDEVICE(REDHAT, 0x0010), /* Qemu emulated controller */
3903	.driver_data = NVME_QUIRK_BOGUS_NID, },
3904	{ PCI_DEVICE(0x1217, 0x8760), /* O2 Micro 64GB Steam Deck */
3905	.driver_data = NVME_QUIRK_DMAPOOL_ALIGN_512, },
3906	{ PCI_DEVICE(0x126f, 0x1001), /* Silicon Motion generic */
3907	.driver_data = NVME_QUIRK_NO_DEEPEST_PS |
3908	NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3909	{ PCI_DEVICE(0x126f, 0x2262), /* Silicon Motion generic */
3910	.driver_data = NVME_QUIRK_NO_DEEPEST_PS |
3911	NVME_QUIRK_BOGUS_NID, },
3912	{ PCI_DEVICE(0x126f, 0x2263), /* Silicon Motion unidentified */
3913	.driver_data = NVME_QUIRK_NO_NS_DESC_LIST |
3914	NVME_QUIRK_BOGUS_NID, },
3915	{ PCI_DEVICE(0x1bb1, 0x0100), /* Seagate Nytro Flash Storage */
3916	.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
3917	NVME_QUIRK_NO_NS_DESC_LIST, },
3918	{ PCI_DEVICE(0x1c58, 0x0003), /* HGST adapter */
3919	.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3920	{ PCI_DEVICE(0x1c58, 0x0023), /* WDC SN200 adapter */
3921	.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3922	{ PCI_DEVICE(0x1c5f, 0x0540), /* Memblaze Pblaze4 adapter */
3923	.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3924	{ PCI_DEVICE(0x144d, 0xa821), /* Samsung PM1725 */
3925	.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3926	{ PCI_DEVICE(0x144d, 0xa822), /* Samsung PM1725a */
3927	.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
3928	NVME_QUIRK_DISABLE_WRITE_ZEROES|
3929	NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3930	{ PCI_DEVICE(0x15b7, 0x5008), /* Sandisk SN530 */
3931	.driver_data = NVME_QUIRK_BROKEN_MSI },
3932	{ PCI_DEVICE(0x15b7, 0x5009), /* Sandisk SN550 */
3933	.driver_data = NVME_QUIRK_BROKEN_MSI |
3934	NVME_QUIRK_NO_DEEPEST_PS },
3935	{ PCI_DEVICE(0x1987, 0x5012), /* Phison E12 */
3936	.driver_data = NVME_QUIRK_BOGUS_NID, },
3937	{ PCI_DEVICE(0x1987, 0x5016), /* Phison E16 */
3938	.driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN |
3939	NVME_QUIRK_BOGUS_NID, },
3940	{ PCI_DEVICE(0x1987, 0x5019), /* phison E19 */
3941	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3942	{ PCI_DEVICE(0x1987, 0x5021), /* Phison E21 */
3943	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3944	{ PCI_DEVICE(0x1b4b, 0x1092), /* Lexar 256 GB SSD */
3945	.driver_data = NVME_QUIRK_NO_NS_DESC_LIST |
3946	NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3947	{ PCI_DEVICE(0x1cc1, 0x33f8), /* ADATA IM2P33F8ABR1 1 TB */
3948	.driver_data = NVME_QUIRK_BOGUS_NID, },
3949	{ PCI_DEVICE(0x10ec, 0x5762), /* ADATA SX6000LNP */
3950	.driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN |
3951	NVME_QUIRK_BOGUS_NID, },
3952	{ PCI_DEVICE(0x10ec, 0x5763), /* ADATA SX6000PNP */
3953	.driver_data = NVME_QUIRK_BOGUS_NID, },
3954	{ PCI_DEVICE(0x1cc1, 0x8201), /* ADATA SX8200PNP 512GB */
3955	.driver_data = NVME_QUIRK_NO_DEEPEST_PS |
3956	NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3957	{ PCI_DEVICE(0x1344, 0x5407), /* Micron Technology Inc NVMe SSD */
3958	.driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN },
3959	{ PCI_DEVICE(0x1344, 0x6001), /* Micron Nitro NVMe */
3960	.driver_data = NVME_QUIRK_BOGUS_NID, },
3961	{ PCI_DEVICE(0x1c5c, 0x1504), /* SK Hynix PC400 */
3962	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3963	{ PCI_DEVICE(0x1c5c, 0x174a), /* SK Hynix P31 SSD */
3964	.driver_data = NVME_QUIRK_BOGUS_NID, },
3965	{ PCI_DEVICE(0x1c5c, 0x1D59), /* SK Hynix BC901 */
3966	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3967	{ PCI_DEVICE(0x15b7, 0x2001), /* Sandisk Skyhawk */
3968	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3969	{ PCI_DEVICE(0x1d97, 0x2263), /* SPCC */
3970	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3971	{ PCI_DEVICE(0x144d, 0xa80b), /* Samsung PM9B1 256G and 512G */
3972	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES |
3973	NVME_QUIRK_BOGUS_NID, },
3974	{ PCI_DEVICE(0x144d, 0xa809), /* Samsung MZALQ256HBJD 256G */
3975	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3976	{ PCI_DEVICE(0x144d, 0xa802), /* Samsung SM953 */
3977	.driver_data = NVME_QUIRK_BOGUS_NID, },
3978	{ PCI_DEVICE(0x1cc4, 0x6303), /* UMIS RPJTJ512MGE1QDY 512G */
3979	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3980	{ PCI_DEVICE(0x1cc4, 0x6302), /* UMIS RPJTJ256MGE1QDY 256G */
3981	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3982	{ PCI_DEVICE(0x2646, 0x2262), /* KINGSTON SKC2000 NVMe SSD */
3983	.driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
3984	{ PCI_DEVICE(0x2646, 0x2263), /* KINGSTON A2000 NVMe SSD */
3985	.driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
3986	{ PCI_DEVICE(0x2646, 0x5013), /* Kingston KC3000, Kingston FURY Renegade */
3987	.driver_data = NVME_QUIRK_NO_SECONDARY_TEMP_THRESH, },
3988	{ PCI_DEVICE(0x2646, 0x5018), /* KINGSTON OM8SFP4xxxxP OS21012 NVMe SSD */
3989	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3990	{ PCI_DEVICE(0x2646, 0x5016), /* KINGSTON OM3PGP4xxxxP OS21011 NVMe SSD */
3991	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3992	{ PCI_DEVICE(0x2646, 0x501A), /* KINGSTON OM8PGP4xxxxP OS21005 NVMe SSD */
3993	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3994	{ PCI_DEVICE(0x2646, 0x501B), /* KINGSTON OM8PGP4xxxxQ OS21005 NVMe SSD */
3995	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3996	{ PCI_DEVICE(0x2646, 0x501E), /* KINGSTON OM3PGP4xxxxQ OS21011 NVMe SSD */
3997	.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3998	{ PCI_DEVICE(0x1f40, 0x1202), /* Netac Technologies Co. NV3000 NVMe SSD */
3999	.driver_data = NVME_QUIRK_BOGUS_NID, },
4000	{ PCI_DEVICE(0x1f40, 0x5236), /* Netac Technologies Co. NV7000 NVMe SSD */
4001	.driver_data = NVME_QUIRK_BOGUS_NID, },
4002	{ PCI_DEVICE(0x1e4B, 0x1001), /* MAXIO MAP1001 */
4003	.driver_data = NVME_QUIRK_BOGUS_NID, },
4004	{ PCI_DEVICE(0x1e4B, 0x1002), /* MAXIO MAP1002 */
4005	.driver_data = NVME_QUIRK_BOGUS_NID, },
4006	{ PCI_DEVICE(0x1e4B, 0x1202), /* MAXIO MAP1202 */
4007	.driver_data = NVME_QUIRK_BOGUS_NID, },
4008	{ PCI_DEVICE(0x1e4B, 0x1602), /* MAXIO MAP1602 */
4009	.driver_data = NVME_QUIRK_BOGUS_NID, },
4010	{ PCI_DEVICE(0x1cc1, 0x5350), /* ADATA XPG GAMMIX S50 */
4011	.driver_data = NVME_QUIRK_BOGUS_NID, },
4012	{ PCI_DEVICE(0x1dbe, 0x5216), /* Acer/INNOGRIT FA100/5216 NVMe SSD */
4013	.driver_data = NVME_QUIRK_BOGUS_NID, },
4014	{ PCI_DEVICE(0x1dbe, 0x5236), /* ADATA XPG GAMMIX S70 */
4015	.driver_data = NVME_QUIRK_BOGUS_NID, },
4016	{ PCI_DEVICE(0x1e49, 0x0021), /* ZHITAI TiPro5000 NVMe SSD */
4017	.driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
4018	{ PCI_DEVICE(0x1e49, 0x0041), /* ZHITAI TiPro7000 NVMe SSD */
4019	.driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
4020	{ PCI_DEVICE(0x1fa0, 0x2283), /* Wodposit WPBSNM8-256GTP */
4021	.driver_data = NVME_QUIRK_NO_SECONDARY_TEMP_THRESH, },
4022	{ PCI_DEVICE(0x025e, 0xf1ac), /* SOLIDIGM P44 pro SSDPFKKW020X7 */
4023	.driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
4024	{ PCI_DEVICE(0xc0a9, 0x540a), /* Crucial P2 */
4025	.driver_data = NVME_QUIRK_BOGUS_NID, },
4026	{ PCI_DEVICE(0x1d97, 0x2263), /* Lexar NM610 */
4027	.driver_data = NVME_QUIRK_BOGUS_NID, },
4028	{ PCI_DEVICE(0x1d97, 0x1d97), /* Lexar NM620 */
4029	.driver_data = NVME_QUIRK_BOGUS_NID, },
4030	{ PCI_DEVICE(0x1d97, 0x2269), /* Lexar NM760 */
4031	.driver_data = NVME_QUIRK_BOGUS_NID |
4032	NVME_QUIRK_IGNORE_DEV_SUBNQN, },
4033	{ PCI_DEVICE(0x10ec, 0x5763), /* TEAMGROUP T-FORCE CARDEA ZERO Z330 SSD */
4034	.driver_data = NVME_QUIRK_BOGUS_NID, },
4035	{ PCI_DEVICE(0x1e4b, 0x1602), /* HS-SSD-FUTURE 2048G */
4036	.driver_data = NVME_QUIRK_BOGUS_NID, },
4037	{ PCI_DEVICE(0x10ec, 0x5765), /* TEAMGROUP MP33 2TB SSD */
4038	.driver_data = NVME_QUIRK_BOGUS_NID, },
4039	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0061),
4040	.driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
4041	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0065),
4042	.driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
4043	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x8061),
4044	.driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
4045	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd00),
4046	.driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
4047	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd01),
4048	.driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
4049	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd02),
4050	.driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
4051	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001),
4052	/*
4053	* Fix for the Apple controller found in the MacBook8,1 and
4054	* some MacBook7,1 to avoid controller resets and data loss.
4055	*/
4056	.driver_data = NVME_QUIRK_SINGLE_VECTOR |
4057	NVME_QUIRK_QDEPTH_ONE },
4058	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) },
4059	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2005),
4060	.driver_data = NVME_QUIRK_SINGLE_VECTOR |
4061	NVME_QUIRK_128_BYTES_SQES |
4062	NVME_QUIRK_SHARED_TAGS |
4063	NVME_QUIRK_SKIP_CID_GEN |
4064	NVME_QUIRK_IDENTIFY_CNS },
4065	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
4066	{ 0, }
4067	};
4068	MODULE_DEVICE_TABLE(pci, nvme_id_table);
4069
4070	static struct pci_driver nvme_driver = {
4071	.name = "nvme",
4072	.id_table = nvme_id_table,
4073	.probe = nvme_probe,
4074	.remove = nvme_remove,
4075	.shutdown = nvme_shutdown,
4076	.driver = {
4077	.probe_type = PROBE_PREFER_ASYNCHRONOUS,
4078	#ifdef CONFIG_PM_SLEEP
4079	.pm = &nvme_dev_pm_ops,
4080	#endif
4081	},
4082	.sriov_configure = pci_sriov_configure_simple,
4083	.err_handler = &nvme_err_handler,
4084	};
4085
4086	static int __init nvme_init(void)
4087	{
4088	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
4089	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
4090	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
4091	BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2);
4092
4093	return pci_register_driver(&nvme_driver);
4094	}
4095
4096	static void __exit nvme_exit(void)
4097	{
4098	pci_unregister_driver(dev: &nvme_driver);
4099	flush_workqueue(nvme_wq);
4100	}
4101
4102	MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
4103	MODULE_LICENSE("GPL");
4104	MODULE_VERSION("1.0");
4105	MODULE_DESCRIPTION("NVMe host PCIe transport driver");
4106	module_init(nvme_init);
4107	module_exit(nvme_exit);
4108