1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright (c) 2016 Avago Technologies. All rights reserved.
4	*/
5	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6	#include <linux/module.h>
7	#include <linux/parser.h>
8	#include <uapi/scsi/fc/fc_fs.h>
9	#include <uapi/scsi/fc/fc_els.h>
10	#include <linux/delay.h>
11	#include <linux/overflow.h>
12	#include <linux/blk-cgroup.h>
13	#include "nvme.h"
14	#include "fabrics.h"
15	#include <linux/nvme-fc-driver.h>
16	#include <linux/nvme-fc.h>
17	#include "fc.h"
18	#include <scsi/scsi_transport_fc.h>
19
20	/* *************************** Data Structures/Defines ****************** */
21
22
23	enum nvme_fc_queue_flags {
24	NVME_FC_Q_CONNECTED = 0,
25	NVME_FC_Q_LIVE,
26	};
27
28	#define NVME_FC_DEFAULT_DEV_LOSS_TMO 60 /* seconds */
29	#define NVME_FC_DEFAULT_RECONNECT_TMO 2 /* delay between reconnects
30	* when connected and a
31	* connection failure.
32	*/
33
34	struct nvme_fc_queue {
35	struct nvme_fc_ctrl *ctrl;
36	struct device *dev;
37	struct blk_mq_hw_ctx *hctx;
38	void *lldd_handle;
39	size_t cmnd_capsule_len;
40	u32 qnum;
41	u32 rqcnt;
42	u32 seqno;
43
44	u64 connection_id;
45	atomic_t csn;
46
47	unsigned long flags;
48	} __aligned(sizeof(u64)); /* alignment for other things alloc'd with */
49
50	enum nvme_fcop_flags {
51	FCOP_FLAGS_TERMIO = (1 << 0),
52	FCOP_FLAGS_AEN = (1 << 1),
53	};
54
55	struct nvmefc_ls_req_op {
56	struct nvmefc_ls_req ls_req;
57
58	struct nvme_fc_rport *rport;
59	struct nvme_fc_queue *queue;
60	struct request *rq;
61	u32 flags;
62
63	int ls_error;
64	struct completion ls_done;
65	struct list_head lsreq_list; /* rport->ls_req_list */
66	bool req_queued;
67	};
68
69	struct nvmefc_ls_rcv_op {
70	struct nvme_fc_rport *rport;
71	struct nvmefc_ls_rsp *lsrsp;
72	union nvmefc_ls_requests *rqstbuf;
73	union nvmefc_ls_responses *rspbuf;
74	u16 rqstdatalen;
75	bool handled;
76	dma_addr_t rspdma;
77	struct list_head lsrcv_list; /* rport->ls_rcv_list */
78	} __aligned(sizeof(u64)); /* alignment for other things alloc'd with */
79
80	enum nvme_fcpop_state {
81	FCPOP_STATE_UNINIT = 0,
82	FCPOP_STATE_IDLE = 1,
83	FCPOP_STATE_ACTIVE = 2,
84	FCPOP_STATE_ABORTED = 3,
85	FCPOP_STATE_COMPLETE = 4,
86	};
87
88	struct nvme_fc_fcp_op {
89	struct nvme_request nreq; /*
90	* nvme/host/core.c
91	* requires this to be
92	* the 1st element in the
93	* private structure
94	* associated with the
95	* request.
96	*/
97	struct nvmefc_fcp_req fcp_req;
98
99	struct nvme_fc_ctrl *ctrl;
100	struct nvme_fc_queue *queue;
101	struct request *rq;
102
103	atomic_t state;
104	u32 flags;
105	u32 rqno;
106	u32 nents;
107
108	struct nvme_fc_cmd_iu cmd_iu;
109	struct nvme_fc_ersp_iu rsp_iu;
110	};
111
112	struct nvme_fcp_op_w_sgl {
113	struct nvme_fc_fcp_op op;
114	struct scatterlist sgl[NVME_INLINE_SG_CNT];
115	uint8_t priv[];
116	};
117
118	struct nvme_fc_lport {
119	struct nvme_fc_local_port localport;
120
121	struct ida endp_cnt;
122	struct list_head port_list; /* nvme_fc_port_list */
123	struct list_head endp_list;
124	struct device *dev; /* physical device for dma */
125	struct nvme_fc_port_template *ops;
126	struct kref ref;
127	atomic_t act_rport_cnt;
128	} __aligned(sizeof(u64)); /* alignment for other things alloc'd with */
129
130	struct nvme_fc_rport {
131	struct nvme_fc_remote_port remoteport;
132
133	struct list_head endp_list; /* for lport->endp_list */
134	struct list_head ctrl_list;
135	struct list_head ls_req_list;
136	struct list_head ls_rcv_list;
137	struct list_head disc_list;
138	struct device *dev; /* physical device for dma */
139	struct nvme_fc_lport *lport;
140	spinlock_t lock;
141	struct kref ref;
142	atomic_t act_ctrl_cnt;
143	unsigned long dev_loss_end;
144	struct work_struct lsrcv_work;
145	} __aligned(sizeof(u64)); /* alignment for other things alloc'd with */
146
147	/* fc_ctrl flags values - specified as bit positions */
148	#define ASSOC_ACTIVE 0
149	#define ASSOC_FAILED 1
150	#define FCCTRL_TERMIO 2
151
152	struct nvme_fc_ctrl {
153	spinlock_t lock;
154	struct nvme_fc_queue *queues;
155	struct device *dev;
156	struct nvme_fc_lport *lport;
157	struct nvme_fc_rport *rport;
158	u32 cnum;
159
160	bool ioq_live;
161	u64 association_id;
162	struct nvmefc_ls_rcv_op *rcv_disconn;
163
164	struct list_head ctrl_list; /* rport->ctrl_list */
165
166	struct blk_mq_tag_set admin_tag_set;
167	struct blk_mq_tag_set tag_set;
168
169	struct work_struct ioerr_work;
170	struct delayed_work connect_work;
171
172	struct kref ref;
173	unsigned long flags;
174	u32 iocnt;
175	wait_queue_head_t ioabort_wait;
176
177	struct nvme_fc_fcp_op aen_ops[NVME_NR_AEN_COMMANDS];
178
179	struct nvme_ctrl ctrl;
180	};
181
182	static inline struct nvme_fc_ctrl *
183	to_fc_ctrl(struct nvme_ctrl *ctrl)
184	{
185	return container_of(ctrl, struct nvme_fc_ctrl, ctrl);
186	}
187
188	static inline struct nvme_fc_lport *
189	localport_to_lport(struct nvme_fc_local_port *portptr)
190	{
191	return container_of(portptr, struct nvme_fc_lport, localport);
192	}
193
194	static inline struct nvme_fc_rport *
195	remoteport_to_rport(struct nvme_fc_remote_port *portptr)
196	{
197	return container_of(portptr, struct nvme_fc_rport, remoteport);
198	}
199
200	static inline struct nvmefc_ls_req_op *
201	ls_req_to_lsop(struct nvmefc_ls_req *lsreq)
202	{
203	return container_of(lsreq, struct nvmefc_ls_req_op, ls_req);
204	}
205
206	static inline struct nvme_fc_fcp_op *
207	fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq)
208	{
209	return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req);
210	}
211
212
213
214	/* *************************** Globals **************************** */
215
216
217	static DEFINE_SPINLOCK(nvme_fc_lock);
218
219	static LIST_HEAD(nvme_fc_lport_list);
220	static DEFINE_IDA(nvme_fc_local_port_cnt);
221	static DEFINE_IDA(nvme_fc_ctrl_cnt);
222
223	/*
224	* These items are short-term. They will eventually be moved into
225	* a generic FC class. See comments in module init.
226	*/
227	static struct device *fc_udev_device;
228
229	static void nvme_fc_complete_rq(struct request *rq);
230
231	/* *********************** FC-NVME Port Management ************************ */
232
233	static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *,
234	struct nvme_fc_queue *, unsigned int);
235
236	static void nvme_fc_handle_ls_rqst_work(struct work_struct *work);
237
238
239	static void
240	nvme_fc_free_lport(struct kref *ref)
241	{
242	struct nvme_fc_lport *lport =
243	container_of(ref, struct nvme_fc_lport, ref);
244	unsigned long flags;
245
246	WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
247	WARN_ON(!list_empty(&lport->endp_list));
248
249	/* remove from transport list */
250	spin_lock_irqsave(&nvme_fc_lock, flags);
251	list_del(entry: &lport->port_list);
252	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
253
254	ida_free(&nvme_fc_local_port_cnt, id: lport->localport.port_num);
255	ida_destroy(ida: &lport->endp_cnt);
256
257	put_device(dev: lport->dev);
258
259	kfree(objp: lport);
260	}
261
262	static void
263	nvme_fc_lport_put(struct nvme_fc_lport *lport)
264	{
265	kref_put(kref: &lport->ref, release: nvme_fc_free_lport);
266	}
267
268	static int
269	nvme_fc_lport_get(struct nvme_fc_lport *lport)
270	{
271	return kref_get_unless_zero(kref: &lport->ref);
272	}
273
274
275	static struct nvme_fc_lport *
276	nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo,
277	struct nvme_fc_port_template *ops,
278	struct device *dev)
279	{
280	struct nvme_fc_lport *lport;
281	unsigned long flags;
282
283	spin_lock_irqsave(&nvme_fc_lock, flags);
284
285	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
286	if (lport->localport.node_name != pinfo->node_name ||
287	lport->localport.port_name != pinfo->port_name)
288	continue;
289
290	if (lport->dev != dev) {
291	lport = ERR_PTR(error: -EXDEV);
292	goto out_done;
293	}
294
295	if (lport->localport.port_state != FC_OBJSTATE_DELETED) {
296	lport = ERR_PTR(error: -EEXIST);
297	goto out_done;
298	}
299
300	if (!nvme_fc_lport_get(lport)) {
301	/*
302	* fails if ref cnt already 0. If so,
303	* act as if lport already deleted
304	*/
305	lport = NULL;
306	goto out_done;
307	}
308
309	/* resume the lport */
310
311	lport->ops = ops;
312	lport->localport.port_role = pinfo->port_role;
313	lport->localport.port_id = pinfo->port_id;
314	lport->localport.port_state = FC_OBJSTATE_ONLINE;
315
316	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
317
318	return lport;
319	}
320
321	lport = NULL;
322
323	out_done:
324	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
325
326	return lport;
327	}
328
329	/**
330	* nvme_fc_register_localport - transport entry point called by an
331	* LLDD to register the existence of a NVME
332	* host FC port.
333	* @pinfo: pointer to information about the port to be registered
334	* @template: LLDD entrypoints and operational parameters for the port
335	* @dev: physical hardware device node port corresponds to. Will be
336	* used for DMA mappings
337	* @portptr: pointer to a local port pointer. Upon success, the routine
338	* will allocate a nvme_fc_local_port structure and place its
339	* address in the local port pointer. Upon failure, local port
340	* pointer will be set to 0.
341	*
342	* Returns:
343	* a completion status. Must be 0 upon success; a negative errno
344	* (ex: -ENXIO) upon failure.
345	*/
346	int
347	nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
348	struct nvme_fc_port_template *template,
349	struct device *dev,
350	struct nvme_fc_local_port **portptr)
351	{
352	struct nvme_fc_lport *newrec;
353	unsigned long flags;
354	int ret, idx;
355
356	if (!template->localport_delete || !template->remoteport_delete ||
357	!template->ls_req || !template->fcp_io ||
358	!template->ls_abort || !template->fcp_abort ||
359	!template->max_hw_queues || !template->max_sgl_segments ||
360	!template->max_dif_sgl_segments || !template->dma_boundary) {
361	ret = -EINVAL;
362	goto out_reghost_failed;
363	}
364
365	/*
366	* look to see if there is already a localport that had been
367	* deregistered and in the process of waiting for all the
368	* references to fully be removed. If the references haven't
369	* expired, we can simply re-enable the localport. Remoteports
370	* and controller reconnections should resume naturally.
371	*/
372	newrec = nvme_fc_attach_to_unreg_lport(pinfo, ops: template, dev);
373
374	/* found an lport, but something about its state is bad */
375	if (IS_ERR(ptr: newrec)) {
376	ret = PTR_ERR(ptr: newrec);
377	goto out_reghost_failed;
378
379	/* found existing lport, which was resumed */
380	} else if (newrec) {
381	*portptr = &newrec->localport;
382	return 0;
383	}
384
385	/* nothing found - allocate a new localport struct */
386
387	newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz),
388	GFP_KERNEL);
389	if (!newrec) {
390	ret = -ENOMEM;
391	goto out_reghost_failed;
392	}
393
394	idx = ida_alloc(ida: &nvme_fc_local_port_cnt, GFP_KERNEL);
395	if (idx < 0) {
396	ret = -ENOSPC;
397	goto out_fail_kfree;
398	}
399
400	if (!get_device(dev) && dev) {
401	ret = -ENODEV;
402	goto out_ida_put;
403	}
404
405	INIT_LIST_HEAD(list: &newrec->port_list);
406	INIT_LIST_HEAD(list: &newrec->endp_list);
407	kref_init(kref: &newrec->ref);
408	atomic_set(v: &newrec->act_rport_cnt, i: 0);
409	newrec->ops = template;
410	newrec->dev = dev;
411	ida_init(ida: &newrec->endp_cnt);
412	if (template->local_priv_sz)
413	newrec->localport.private = &newrec[1];
414	else
415	newrec->localport.private = NULL;
416	newrec->localport.node_name = pinfo->node_name;
417	newrec->localport.port_name = pinfo->port_name;
418	newrec->localport.port_role = pinfo->port_role;
419	newrec->localport.port_id = pinfo->port_id;
420	newrec->localport.port_state = FC_OBJSTATE_ONLINE;
421	newrec->localport.port_num = idx;
422
423	spin_lock_irqsave(&nvme_fc_lock, flags);
424	list_add_tail(new: &newrec->port_list, head: &nvme_fc_lport_list);
425	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
426
427	if (dev)
428	dma_set_seg_boundary(dev, mask: template->dma_boundary);
429
430	*portptr = &newrec->localport;
431	return 0;
432
433	out_ida_put:
434	ida_free(&nvme_fc_local_port_cnt, id: idx);
435	out_fail_kfree:
436	kfree(objp: newrec);
437	out_reghost_failed:
438	*portptr = NULL;
439
440	return ret;
441	}
442	EXPORT_SYMBOL_GPL(nvme_fc_register_localport);
443
444	/**
445	* nvme_fc_unregister_localport - transport entry point called by an
446	* LLDD to deregister/remove a previously
447	* registered a NVME host FC port.
448	* @portptr: pointer to the (registered) local port that is to be deregistered.
449	*
450	* Returns:
451	* a completion status. Must be 0 upon success; a negative errno
452	* (ex: -ENXIO) upon failure.
453	*/
454	int
455	nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr)
456	{
457	struct nvme_fc_lport *lport = localport_to_lport(portptr);
458	unsigned long flags;
459
460	if (!portptr)
461	return -EINVAL;
462
463	spin_lock_irqsave(&nvme_fc_lock, flags);
464
465	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
466	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
467	return -EINVAL;
468	}
469	portptr->port_state = FC_OBJSTATE_DELETED;
470
471	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
472
473	if (atomic_read(v: &lport->act_rport_cnt) == 0)
474	lport->ops->localport_delete(&lport->localport);
475
476	nvme_fc_lport_put(lport);
477
478	return 0;
479	}
480	EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport);
481
482	/*
483	* TRADDR strings, per FC-NVME are fixed format:
484	* "nn-0x<16hexdigits>:pn-0x<16hexdigits>" - 43 characters
485	* udev event will only differ by prefix of what field is
486	* being specified:
487	* "NVMEFC_HOST_TRADDR=" or "NVMEFC_TRADDR=" - 19 max characters
488	* 19 + 43 + null_fudge = 64 characters
489	*/
490	#define FCNVME_TRADDR_LENGTH 64
491
492	static void
493	nvme_fc_signal_discovery_scan(struct nvme_fc_lport *lport,
494	struct nvme_fc_rport *rport)
495	{
496	char hostaddr[FCNVME_TRADDR_LENGTH]; /* NVMEFC_HOST_TRADDR=...*/
497	char tgtaddr[FCNVME_TRADDR_LENGTH]; /* NVMEFC_TRADDR=...*/
498	char *envp[4] = { "FC_EVENT=nvmediscovery", hostaddr, tgtaddr, NULL };
499
500	if (!(rport->remoteport.port_role & FC_PORT_ROLE_NVME_DISCOVERY))
501	return;
502
503	snprintf(buf: hostaddr, size: sizeof(hostaddr),
504	fmt: "NVMEFC_HOST_TRADDR=nn-0x%016llx:pn-0x%016llx",
505	lport->localport.node_name, lport->localport.port_name);
506	snprintf(buf: tgtaddr, size: sizeof(tgtaddr),
507	fmt: "NVMEFC_TRADDR=nn-0x%016llx:pn-0x%016llx",
508	rport->remoteport.node_name, rport->remoteport.port_name);
509	kobject_uevent_env(kobj: &fc_udev_device->kobj, action: KOBJ_CHANGE, envp);
510	}
511
512	static void
513	nvme_fc_free_rport(struct kref *ref)
514	{
515	struct nvme_fc_rport *rport =
516	container_of(ref, struct nvme_fc_rport, ref);
517	struct nvme_fc_lport *lport =
518	localport_to_lport(portptr: rport->remoteport.localport);
519	unsigned long flags;
520
521	WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED);
522	WARN_ON(!list_empty(&rport->ctrl_list));
523	WARN_ON(!list_empty(&rport->ls_req_list));
524	WARN_ON(!list_empty(&rport->ls_rcv_list));
525
526	/* remove from lport list */
527	spin_lock_irqsave(&nvme_fc_lock, flags);
528	list_del(entry: &rport->endp_list);
529	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
530
531	WARN_ON(!list_empty(&rport->disc_list));
532	ida_free(&lport->endp_cnt, id: rport->remoteport.port_num);
533
534	kfree(objp: rport);
535
536	nvme_fc_lport_put(lport);
537	}
538
539	static void
540	nvme_fc_rport_put(struct nvme_fc_rport *rport)
541	{
542	kref_put(kref: &rport->ref, release: nvme_fc_free_rport);
543	}
544
545	static int
546	nvme_fc_rport_get(struct nvme_fc_rport *rport)
547	{
548	return kref_get_unless_zero(kref: &rport->ref);
549	}
550
551	static void
552	nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
553	{
554	switch (nvme_ctrl_state(ctrl: &ctrl->ctrl)) {
555	case NVME_CTRL_NEW:
556	case NVME_CTRL_CONNECTING:
557	/*
558	* As all reconnects were suppressed, schedule a
559	* connect.
560	*/
561	dev_info(ctrl->ctrl.device,
562	"NVME-FC{%d}: connectivity re-established. "
563	"Attempting reconnect\n", ctrl->cnum);
564
565	queue_delayed_work(wq: nvme_wq, dwork: &ctrl->connect_work, delay: 0);
566	break;
567
568	case NVME_CTRL_RESETTING:
569	/*
570	* Controller is already in the process of terminating the
571	* association. No need to do anything further. The reconnect
572	* step will naturally occur after the reset completes.
573	*/
574	break;
575
576	default:
577	/* no action to take - let it delete */
578	break;
579	}
580	}
581
582	static struct nvme_fc_rport *
583	nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport *lport,
584	struct nvme_fc_port_info *pinfo)
585	{
586	struct nvme_fc_rport *rport;
587	struct nvme_fc_ctrl *ctrl;
588	unsigned long flags;
589
590	spin_lock_irqsave(&nvme_fc_lock, flags);
591
592	list_for_each_entry(rport, &lport->endp_list, endp_list) {
593	if (rport->remoteport.node_name != pinfo->node_name ||
594	rport->remoteport.port_name != pinfo->port_name)
595	continue;
596
597	if (!nvme_fc_rport_get(rport)) {
598	rport = ERR_PTR(error: -ENOLCK);
599	goto out_done;
600	}
601
602	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
603
604	spin_lock_irqsave(&rport->lock, flags);
605
606	/* has it been unregistered */
607	if (rport->remoteport.port_state != FC_OBJSTATE_DELETED) {
608	/* means lldd called us twice */
609	spin_unlock_irqrestore(lock: &rport->lock, flags);
610	nvme_fc_rport_put(rport);
611	return ERR_PTR(error: -ESTALE);
612	}
613
614	rport->remoteport.port_role = pinfo->port_role;
615	rport->remoteport.port_id = pinfo->port_id;
616	rport->remoteport.port_state = FC_OBJSTATE_ONLINE;
617	rport->dev_loss_end = 0;
618
619	/*
620	* kick off a reconnect attempt on all associations to the
621	* remote port. A successful reconnects will resume i/o.
622	*/
623	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list)
624	nvme_fc_resume_controller(ctrl);
625
626	spin_unlock_irqrestore(lock: &rport->lock, flags);
627
628	return rport;
629	}
630
631	rport = NULL;
632
633	out_done:
634	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
635
636	return rport;
637	}
638
639	static inline void
640	__nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport,
641	struct nvme_fc_port_info *pinfo)
642	{
643	if (pinfo->dev_loss_tmo)
644	rport->remoteport.dev_loss_tmo = pinfo->dev_loss_tmo;
645	else
646	rport->remoteport.dev_loss_tmo = NVME_FC_DEFAULT_DEV_LOSS_TMO;
647	}
648
649	/**
650	* nvme_fc_register_remoteport - transport entry point called by an
651	* LLDD to register the existence of a NVME
652	* subsystem FC port on its fabric.
653	* @localport: pointer to the (registered) local port that the remote
654	* subsystem port is connected to.
655	* @pinfo: pointer to information about the port to be registered
656	* @portptr: pointer to a remote port pointer. Upon success, the routine
657	* will allocate a nvme_fc_remote_port structure and place its
658	* address in the remote port pointer. Upon failure, remote port
659	* pointer will be set to 0.
660	*
661	* Returns:
662	* a completion status. Must be 0 upon success; a negative errno
663	* (ex: -ENXIO) upon failure.
664	*/
665	int
666	nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
667	struct nvme_fc_port_info *pinfo,
668	struct nvme_fc_remote_port **portptr)
669	{
670	struct nvme_fc_lport *lport = localport_to_lport(portptr: localport);
671	struct nvme_fc_rport *newrec;
672	unsigned long flags;
673	int ret, idx;
674
675	if (!nvme_fc_lport_get(lport)) {
676	ret = -ESHUTDOWN;
677	goto out_reghost_failed;
678	}
679
680	/*
681	* look to see if there is already a remoteport that is waiting
682	* for a reconnect (within dev_loss_tmo) with the same WWN's.
683	* If so, transition to it and reconnect.
684	*/
685	newrec = nvme_fc_attach_to_suspended_rport(lport, pinfo);
686
687	/* found an rport, but something about its state is bad */
688	if (IS_ERR(ptr: newrec)) {
689	ret = PTR_ERR(ptr: newrec);
690	goto out_lport_put;
691
692	/* found existing rport, which was resumed */
693	} else if (newrec) {
694	nvme_fc_lport_put(lport);
695	__nvme_fc_set_dev_loss_tmo(rport: newrec, pinfo);
696	nvme_fc_signal_discovery_scan(lport, rport: newrec);
697	*portptr = &newrec->remoteport;
698	return 0;
699	}
700
701	/* nothing found - allocate a new remoteport struct */
702
703	newrec = kmalloc((sizeof(*newrec) + lport->ops->remote_priv_sz),
704	GFP_KERNEL);
705	if (!newrec) {
706	ret = -ENOMEM;
707	goto out_lport_put;
708	}
709
710	idx = ida_alloc(ida: &lport->endp_cnt, GFP_KERNEL);
711	if (idx < 0) {
712	ret = -ENOSPC;
713	goto out_kfree_rport;
714	}
715
716	INIT_LIST_HEAD(list: &newrec->endp_list);
717	INIT_LIST_HEAD(list: &newrec->ctrl_list);
718	INIT_LIST_HEAD(list: &newrec->ls_req_list);
719	INIT_LIST_HEAD(list: &newrec->disc_list);
720	kref_init(kref: &newrec->ref);
721	atomic_set(v: &newrec->act_ctrl_cnt, i: 0);
722	spin_lock_init(&newrec->lock);
723	newrec->remoteport.localport = &lport->localport;
724	INIT_LIST_HEAD(list: &newrec->ls_rcv_list);
725	newrec->dev = lport->dev;
726	newrec->lport = lport;
727	if (lport->ops->remote_priv_sz)
728	newrec->remoteport.private = &newrec[1];
729	else
730	newrec->remoteport.private = NULL;
731	newrec->remoteport.port_role = pinfo->port_role;
732	newrec->remoteport.node_name = pinfo->node_name;
733	newrec->remoteport.port_name = pinfo->port_name;
734	newrec->remoteport.port_id = pinfo->port_id;
735	newrec->remoteport.port_state = FC_OBJSTATE_ONLINE;
736	newrec->remoteport.port_num = idx;
737	__nvme_fc_set_dev_loss_tmo(rport: newrec, pinfo);
738	INIT_WORK(&newrec->lsrcv_work, nvme_fc_handle_ls_rqst_work);
739
740	spin_lock_irqsave(&nvme_fc_lock, flags);
741	list_add_tail(new: &newrec->endp_list, head: &lport->endp_list);
742	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
743
744	nvme_fc_signal_discovery_scan(lport, rport: newrec);
745
746	*portptr = &newrec->remoteport;
747	return 0;
748
749	out_kfree_rport:
750	kfree(objp: newrec);
751	out_lport_put:
752	nvme_fc_lport_put(lport);
753	out_reghost_failed:
754	*portptr = NULL;
755	return ret;
756	}
757	EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport);
758
759	static int
760	nvme_fc_abort_lsops(struct nvme_fc_rport *rport)
761	{
762	struct nvmefc_ls_req_op *lsop;
763	unsigned long flags;
764
765	restart:
766	spin_lock_irqsave(&rport->lock, flags);
767
768	list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) {
769	if (!(lsop->flags & FCOP_FLAGS_TERMIO)) {
770	lsop->flags |= FCOP_FLAGS_TERMIO;
771	spin_unlock_irqrestore(lock: &rport->lock, flags);
772	rport->lport->ops->ls_abort(&rport->lport->localport,
773	&rport->remoteport,
774	&lsop->ls_req);
775	goto restart;
776	}
777	}
778	spin_unlock_irqrestore(lock: &rport->lock, flags);
779
780	return 0;
781	}
782
783	static void
784	nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
785	{
786	dev_info(ctrl->ctrl.device,
787	"NVME-FC{%d}: controller connectivity lost. Awaiting "
788	"Reconnect", ctrl->cnum);
789
790	set_bit(ASSOC_FAILED, addr: &ctrl->flags);
791	nvme_reset_ctrl(ctrl: &ctrl->ctrl);
792	}
793
794	/**
795	* nvme_fc_unregister_remoteport - transport entry point called by an
796	* LLDD to deregister/remove a previously
797	* registered a NVME subsystem FC port.
798	* @portptr: pointer to the (registered) remote port that is to be
799	* deregistered.
800	*
801	* Returns:
802	* a completion status. Must be 0 upon success; a negative errno
803	* (ex: -ENXIO) upon failure.
804	*/
805	int
806	nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr)
807	{
808	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
809	struct nvme_fc_ctrl *ctrl;
810	unsigned long flags;
811
812	if (!portptr)
813	return -EINVAL;
814
815	spin_lock_irqsave(&rport->lock, flags);
816
817	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
818	spin_unlock_irqrestore(lock: &rport->lock, flags);
819	return -EINVAL;
820	}
821	portptr->port_state = FC_OBJSTATE_DELETED;
822
823	rport->dev_loss_end = jiffies + (portptr->dev_loss_tmo * HZ);
824
825	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
826	/* if dev_loss_tmo==0, dev loss is immediate */
827	if (!portptr->dev_loss_tmo) {
828	dev_warn(ctrl->ctrl.device,
829	"NVME-FC{%d}: controller connectivity lost.\n",
830	ctrl->cnum);
831	nvme_delete_ctrl(ctrl: &ctrl->ctrl);
832	} else
833	nvme_fc_ctrl_connectivity_loss(ctrl);
834	}
835
836	spin_unlock_irqrestore(lock: &rport->lock, flags);
837
838	nvme_fc_abort_lsops(rport);
839
840	if (atomic_read(v: &rport->act_ctrl_cnt) == 0)
841	rport->lport->ops->remoteport_delete(portptr);
842
843	/*
844	* release the reference, which will allow, if all controllers
845	* go away, which should only occur after dev_loss_tmo occurs,
846	* for the rport to be torn down.
847	*/
848	nvme_fc_rport_put(rport);
849
850	return 0;
851	}
852	EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport);
853
854	/**
855	* nvme_fc_rescan_remoteport - transport entry point called by an
856	* LLDD to request a nvme device rescan.
857	* @remoteport: pointer to the (registered) remote port that is to be
858	* rescanned.
859	*
860	* Returns: N/A
861	*/
862	void
863	nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport)
864	{
865	struct nvme_fc_rport *rport = remoteport_to_rport(portptr: remoteport);
866
867	nvme_fc_signal_discovery_scan(lport: rport->lport, rport);
868	}
869	EXPORT_SYMBOL_GPL(nvme_fc_rescan_remoteport);
870
871	int
872	nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *portptr,
873	u32 dev_loss_tmo)
874	{
875	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
876	unsigned long flags;
877
878	spin_lock_irqsave(&rport->lock, flags);
879
880	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
881	spin_unlock_irqrestore(lock: &rport->lock, flags);
882	return -EINVAL;
883	}
884
885	/* a dev_loss_tmo of 0 (immediate) is allowed to be set */
886	rport->remoteport.dev_loss_tmo = dev_loss_tmo;
887
888	spin_unlock_irqrestore(lock: &rport->lock, flags);
889
890	return 0;
891	}
892	EXPORT_SYMBOL_GPL(nvme_fc_set_remoteport_devloss);
893
894
895	/* *********************** FC-NVME DMA Handling **************************** */
896
897	/*
898	* The fcloop device passes in a NULL device pointer. Real LLD's will
899	* pass in a valid device pointer. If NULL is passed to the dma mapping
900	* routines, depending on the platform, it may or may not succeed, and
901	* may crash.
902	*
903	* As such:
904	* Wrap all the dma routines and check the dev pointer.
905	*
906	* If simple mappings (return just a dma address, we'll noop them,
907	* returning a dma address of 0.
908	*
909	* On more complex mappings (dma_map_sg), a pseudo routine fills
910	* in the scatter list, setting all dma addresses to 0.
911	*/
912
913	static inline dma_addr_t
914	fc_dma_map_single(struct device *dev, void *ptr, size_t size,
915	enum dma_data_direction dir)
916	{
917	return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
918	}
919
920	static inline int
921	fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
922	{
923	return dev ? dma_mapping_error(dev, dma_addr) : 0;
924	}
925
926	static inline void
927	fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
928	enum dma_data_direction dir)
929	{
930	if (dev)
931	dma_unmap_single(dev, addr, size, dir);
932	}
933
934	static inline void
935	fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
936	enum dma_data_direction dir)
937	{
938	if (dev)
939	dma_sync_single_for_cpu(dev, addr, size, dir);
940	}
941
942	static inline void
943	fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
944	enum dma_data_direction dir)
945	{
946	if (dev)
947	dma_sync_single_for_device(dev, addr, size, dir);
948	}
949
950	/* pseudo dma_map_sg call */
951	static int
952	fc_map_sg(struct scatterlist *sg, int nents)
953	{
954	struct scatterlist *s;
955	int i;
956
957	WARN_ON(nents == 0 || sg[0].length == 0);
958
959	for_each_sg(sg, s, nents, i) {
960	s->dma_address = 0L;
961	#ifdef CONFIG_NEED_SG_DMA_LENGTH
962	s->dma_length = s->length;
963	#endif
964	}
965	return nents;
966	}
967
968	static inline int
969	fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
970	enum dma_data_direction dir)
971	{
972	return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
973	}
974
975	static inline void
976	fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
977	enum dma_data_direction dir)
978	{
979	if (dev)
980	dma_unmap_sg(dev, sg, nents, dir);
981	}
982
983	/* *********************** FC-NVME LS Handling **************************** */
984
985	static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *);
986	static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);
987
988	static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg);
989
990	static void
991	__nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop)
992	{
993	struct nvme_fc_rport *rport = lsop->rport;
994	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
995	unsigned long flags;
996
997	spin_lock_irqsave(&rport->lock, flags);
998
999	if (!lsop->req_queued) {
1000	spin_unlock_irqrestore(lock: &rport->lock, flags);
1001	return;
1002	}
1003
1004	list_del(entry: &lsop->lsreq_list);
1005
1006	lsop->req_queued = false;
1007
1008	spin_unlock_irqrestore(lock: &rport->lock, flags);
1009
1010	fc_dma_unmap_single(dev: rport->dev, addr: lsreq->rqstdma,
1011	size: (lsreq->rqstlen + lsreq->rsplen),
1012	dir: DMA_BIDIRECTIONAL);
1013
1014	nvme_fc_rport_put(rport);
1015	}
1016
1017	static int
1018	__nvme_fc_send_ls_req(struct nvme_fc_rport *rport,
1019	struct nvmefc_ls_req_op *lsop,
1020	void (*done)(struct nvmefc_ls_req *req, int status))
1021	{
1022	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1023	unsigned long flags;
1024	int ret = 0;
1025
1026	if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
1027	return -ECONNREFUSED;
1028
1029	if (!nvme_fc_rport_get(rport))
1030	return -ESHUTDOWN;
1031
1032	lsreq->done = done;
1033	lsop->rport = rport;
1034	lsop->req_queued = false;
1035	INIT_LIST_HEAD(list: &lsop->lsreq_list);
1036	init_completion(x: &lsop->ls_done);
1037
1038	lsreq->rqstdma = fc_dma_map_single(dev: rport->dev, ptr: lsreq->rqstaddr,
1039	size: lsreq->rqstlen + lsreq->rsplen,
1040	dir: DMA_BIDIRECTIONAL);
1041	if (fc_dma_mapping_error(dev: rport->dev, dma_addr: lsreq->rqstdma)) {
1042	ret = -EFAULT;
1043	goto out_putrport;
1044	}
1045	lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
1046
1047	spin_lock_irqsave(&rport->lock, flags);
1048
1049	list_add_tail(new: &lsop->lsreq_list, head: &rport->ls_req_list);
1050
1051	lsop->req_queued = true;
1052
1053	spin_unlock_irqrestore(lock: &rport->lock, flags);
1054
1055	ret = rport->lport->ops->ls_req(&rport->lport->localport,
1056	&rport->remoteport, lsreq);
1057	if (ret)
1058	goto out_unlink;
1059
1060	return 0;
1061
1062	out_unlink:
1063	lsop->ls_error = ret;
1064	spin_lock_irqsave(&rport->lock, flags);
1065	lsop->req_queued = false;
1066	list_del(entry: &lsop->lsreq_list);
1067	spin_unlock_irqrestore(lock: &rport->lock, flags);
1068	fc_dma_unmap_single(dev: rport->dev, addr: lsreq->rqstdma,
1069	size: (lsreq->rqstlen + lsreq->rsplen),
1070	dir: DMA_BIDIRECTIONAL);
1071	out_putrport:
1072	nvme_fc_rport_put(rport);
1073
1074	return ret;
1075	}
1076
1077	static void
1078	nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status)
1079	{
1080	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1081
1082	lsop->ls_error = status;
1083	complete(&lsop->ls_done);
1084	}
1085
1086	static int
1087	nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop)
1088	{
1089	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1090	struct fcnvme_ls_rjt *rjt = lsreq->rspaddr;
1091	int ret;
1092
1093	ret = __nvme_fc_send_ls_req(rport, lsop, done: nvme_fc_send_ls_req_done);
1094
1095	if (!ret) {
1096	/*
1097	* No timeout/not interruptible as we need the struct
1098	* to exist until the lldd calls us back. Thus mandate
1099	* wait until driver calls back. lldd responsible for
1100	* the timeout action
1101	*/
1102	wait_for_completion(&lsop->ls_done);
1103
1104	__nvme_fc_finish_ls_req(lsop);
1105
1106	ret = lsop->ls_error;
1107	}
1108
1109	if (ret)
1110	return ret;
1111
1112	/* ACC or RJT payload ? */
1113	if (rjt->w0.ls_cmd == FCNVME_LS_RJT)
1114	return -ENXIO;
1115
1116	return 0;
1117	}
1118
1119	static int
1120	nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport,
1121	struct nvmefc_ls_req_op *lsop,
1122	void (*done)(struct nvmefc_ls_req *req, int status))
1123	{
1124	/* don't wait for completion */
1125
1126	return __nvme_fc_send_ls_req(rport, lsop, done);
1127	}
1128
1129	static int
1130	nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
1131	struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio)
1132	{
1133	struct nvmefc_ls_req_op *lsop;
1134	struct nvmefc_ls_req *lsreq;
1135	struct fcnvme_ls_cr_assoc_rqst *assoc_rqst;
1136	struct fcnvme_ls_cr_assoc_acc *assoc_acc;
1137	unsigned long flags;
1138	int ret, fcret = 0;
1139
1140	lsop = kzalloc((sizeof(*lsop) +
1141	sizeof(*assoc_rqst) + sizeof(*assoc_acc) +
1142	ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1143	if (!lsop) {
1144	dev_info(ctrl->ctrl.device,
1145	"NVME-FC{%d}: send Create Association failed: ENOMEM\n",
1146	ctrl->cnum);
1147	ret = -ENOMEM;
1148	goto out_no_memory;
1149	}
1150
1151	assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)&lsop[1];
1152	assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1];
1153	lsreq = &lsop->ls_req;
1154	if (ctrl->lport->ops->lsrqst_priv_sz)
1155	lsreq->private = &assoc_acc[1];
1156	else
1157	lsreq->private = NULL;
1158
1159	assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION;
1160	assoc_rqst->desc_list_len =
1161	cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1162
1163	assoc_rqst->assoc_cmd.desc_tag =
1164	cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD);
1165	assoc_rqst->assoc_cmd.desc_len =
1166	fcnvme_lsdesc_len(
1167	sz: sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1168
1169	assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1170	assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1);
1171	/* Linux supports only Dynamic controllers */
1172	assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff);
1173	uuid_copy(dst: &assoc_rqst->assoc_cmd.hostid, src: &ctrl->ctrl.opts->host->id);
1174	strscpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn,
1175	sizeof(assoc_rqst->assoc_cmd.hostnqn));
1176	strscpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn,
1177	sizeof(assoc_rqst->assoc_cmd.subnqn));
1178
1179	lsop->queue = queue;
1180	lsreq->rqstaddr = assoc_rqst;
1181	lsreq->rqstlen = sizeof(*assoc_rqst);
1182	lsreq->rspaddr = assoc_acc;
1183	lsreq->rsplen = sizeof(*assoc_acc);
1184	lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1185
1186	ret = nvme_fc_send_ls_req(rport: ctrl->rport, lsop);
1187	if (ret)
1188	goto out_free_buffer;
1189
1190	/* process connect LS completion */
1191
1192	/* validate the ACC response */
1193	if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1194	fcret = VERR_LSACC;
1195	else if (assoc_acc->hdr.desc_list_len !=
1196	fcnvme_lsdesc_len(
1197	sz: sizeof(struct fcnvme_ls_cr_assoc_acc)))
1198	fcret = VERR_CR_ASSOC_ACC_LEN;
1199	else if (assoc_acc->hdr.rqst.desc_tag !=
1200	cpu_to_be32(FCNVME_LSDESC_RQST))
1201	fcret = VERR_LSDESC_RQST;
1202	else if (assoc_acc->hdr.rqst.desc_len !=
1203	fcnvme_lsdesc_len(sz: sizeof(struct fcnvme_lsdesc_rqst)))
1204	fcret = VERR_LSDESC_RQST_LEN;
1205	else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION)
1206	fcret = VERR_CR_ASSOC;
1207	else if (assoc_acc->associd.desc_tag !=
1208	cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1209	fcret = VERR_ASSOC_ID;
1210	else if (assoc_acc->associd.desc_len !=
1211	fcnvme_lsdesc_len(
1212	sz: sizeof(struct fcnvme_lsdesc_assoc_id)))
1213	fcret = VERR_ASSOC_ID_LEN;
1214	else if (assoc_acc->connectid.desc_tag !=
1215	cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1216	fcret = VERR_CONN_ID;
1217	else if (assoc_acc->connectid.desc_len !=
1218	fcnvme_lsdesc_len(sz: sizeof(struct fcnvme_lsdesc_conn_id)))
1219	fcret = VERR_CONN_ID_LEN;
1220
1221	if (fcret) {
1222	ret = -EBADF;
1223	dev_err(ctrl->dev,
1224	"q %d Create Association LS failed: %s\n",
1225	queue->qnum, validation_errors[fcret]);
1226	} else {
1227	spin_lock_irqsave(&ctrl->lock, flags);
1228	ctrl->association_id =
1229	be64_to_cpu(assoc_acc->associd.association_id);
1230	queue->connection_id =
1231	be64_to_cpu(assoc_acc->connectid.connection_id);
1232	set_bit(nr: NVME_FC_Q_CONNECTED, addr: &queue->flags);
1233	spin_unlock_irqrestore(lock: &ctrl->lock, flags);
1234	}
1235
1236	out_free_buffer:
1237	kfree(objp: lsop);
1238	out_no_memory:
1239	if (ret)
1240	dev_err(ctrl->dev,
1241	"queue %d connect admin queue failed (%d).\n",
1242	queue->qnum, ret);
1243	return ret;
1244	}
1245
1246	static int
1247	nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
1248	u16 qsize, u16 ersp_ratio)
1249	{
1250	struct nvmefc_ls_req_op *lsop;
1251	struct nvmefc_ls_req *lsreq;
1252	struct fcnvme_ls_cr_conn_rqst *conn_rqst;
1253	struct fcnvme_ls_cr_conn_acc *conn_acc;
1254	int ret, fcret = 0;
1255
1256	lsop = kzalloc((sizeof(*lsop) +
1257	sizeof(*conn_rqst) + sizeof(*conn_acc) +
1258	ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1259	if (!lsop) {
1260	dev_info(ctrl->ctrl.device,
1261	"NVME-FC{%d}: send Create Connection failed: ENOMEM\n",
1262	ctrl->cnum);
1263	ret = -ENOMEM;
1264	goto out_no_memory;
1265	}
1266
1267	conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)&lsop[1];
1268	conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1];
1269	lsreq = &lsop->ls_req;
1270	if (ctrl->lport->ops->lsrqst_priv_sz)
1271	lsreq->private = (void *)&conn_acc[1];
1272	else
1273	lsreq->private = NULL;
1274
1275	conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION;
1276	conn_rqst->desc_list_len = cpu_to_be32(
1277	sizeof(struct fcnvme_lsdesc_assoc_id) +
1278	sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1279
1280	conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1281	conn_rqst->associd.desc_len =
1282	fcnvme_lsdesc_len(
1283	sz: sizeof(struct fcnvme_lsdesc_assoc_id));
1284	conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
1285	conn_rqst->connect_cmd.desc_tag =
1286	cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD);
1287	conn_rqst->connect_cmd.desc_len =
1288	fcnvme_lsdesc_len(
1289	sz: sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1290	conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1291	conn_rqst->connect_cmd.qid = cpu_to_be16(queue->qnum);
1292	conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1);
1293
1294	lsop->queue = queue;
1295	lsreq->rqstaddr = conn_rqst;
1296	lsreq->rqstlen = sizeof(*conn_rqst);
1297	lsreq->rspaddr = conn_acc;
1298	lsreq->rsplen = sizeof(*conn_acc);
1299	lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1300
1301	ret = nvme_fc_send_ls_req(rport: ctrl->rport, lsop);
1302	if (ret)
1303	goto out_free_buffer;
1304
1305	/* process connect LS completion */
1306
1307	/* validate the ACC response */
1308	if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1309	fcret = VERR_LSACC;
1310	else if (conn_acc->hdr.desc_list_len !=
1311	fcnvme_lsdesc_len(sz: sizeof(struct fcnvme_ls_cr_conn_acc)))
1312	fcret = VERR_CR_CONN_ACC_LEN;
1313	else if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST))
1314	fcret = VERR_LSDESC_RQST;
1315	else if (conn_acc->hdr.rqst.desc_len !=
1316	fcnvme_lsdesc_len(sz: sizeof(struct fcnvme_lsdesc_rqst)))
1317	fcret = VERR_LSDESC_RQST_LEN;
1318	else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION)
1319	fcret = VERR_CR_CONN;
1320	else if (conn_acc->connectid.desc_tag !=
1321	cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1322	fcret = VERR_CONN_ID;
1323	else if (conn_acc->connectid.desc_len !=
1324	fcnvme_lsdesc_len(sz: sizeof(struct fcnvme_lsdesc_conn_id)))
1325	fcret = VERR_CONN_ID_LEN;
1326
1327	if (fcret) {
1328	ret = -EBADF;
1329	dev_err(ctrl->dev,
1330	"q %d Create I/O Connection LS failed: %s\n",
1331	queue->qnum, validation_errors[fcret]);
1332	} else {
1333	queue->connection_id =
1334	be64_to_cpu(conn_acc->connectid.connection_id);
1335	set_bit(nr: NVME_FC_Q_CONNECTED, addr: &queue->flags);
1336	}
1337
1338	out_free_buffer:
1339	kfree(objp: lsop);
1340	out_no_memory:
1341	if (ret)
1342	dev_err(ctrl->dev,
1343	"queue %d connect I/O queue failed (%d).\n",
1344	queue->qnum, ret);
1345	return ret;
1346	}
1347
1348	static void
1349	nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
1350	{
1351	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1352
1353	__nvme_fc_finish_ls_req(lsop);
1354
1355	/* fc-nvme initiator doesn't care about success or failure of cmd */
1356
1357	kfree(objp: lsop);
1358	}
1359
1360	/*
1361	* This routine sends a FC-NVME LS to disconnect (aka terminate)
1362	* the FC-NVME Association. Terminating the association also
1363	* terminates the FC-NVME connections (per queue, both admin and io
1364	* queues) that are part of the association. E.g. things are torn
1365	* down, and the related FC-NVME Association ID and Connection IDs
1366	* become invalid.
1367	*
1368	* The behavior of the fc-nvme initiator is such that its
1369	* understanding of the association and connections will implicitly
1370	* be torn down. The action is implicit as it may be due to a loss of
1371	* connectivity with the fc-nvme target, so you may never get a
1372	* response even if you tried. As such, the action of this routine
1373	* is to asynchronously send the LS, ignore any results of the LS, and
1374	* continue on with terminating the association. If the fc-nvme target
1375	* is present and receives the LS, it too can tear down.
1376	*/
1377	static void
1378	nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
1379	{
1380	struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst;
1381	struct fcnvme_ls_disconnect_assoc_acc *discon_acc;
1382	struct nvmefc_ls_req_op *lsop;
1383	struct nvmefc_ls_req *lsreq;
1384	int ret;
1385
1386	lsop = kzalloc((sizeof(*lsop) +
1387	sizeof(*discon_rqst) + sizeof(*discon_acc) +
1388	ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1389	if (!lsop) {
1390	dev_info(ctrl->ctrl.device,
1391	"NVME-FC{%d}: send Disconnect Association "
1392	"failed: ENOMEM\n",
1393	ctrl->cnum);
1394	return;
1395	}
1396
1397	discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1];
1398	discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1];
1399	lsreq = &lsop->ls_req;
1400	if (ctrl->lport->ops->lsrqst_priv_sz)
1401	lsreq->private = (void *)&discon_acc[1];
1402	else
1403	lsreq->private = NULL;
1404
1405	nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc,
1406	association_id: ctrl->association_id);
1407
1408	ret = nvme_fc_send_ls_req_async(rport: ctrl->rport, lsop,
1409	done: nvme_fc_disconnect_assoc_done);
1410	if (ret)
1411	kfree(objp: lsop);
1412	}
1413
1414	static void
1415	nvme_fc_xmt_ls_rsp_free(struct nvmefc_ls_rcv_op *lsop)
1416	{
1417	struct nvme_fc_rport *rport = lsop->rport;
1418	struct nvme_fc_lport *lport = rport->lport;
1419	unsigned long flags;
1420
1421	spin_lock_irqsave(&rport->lock, flags);
1422	list_del(entry: &lsop->lsrcv_list);
1423	spin_unlock_irqrestore(lock: &rport->lock, flags);
1424
1425	fc_dma_sync_single_for_cpu(dev: lport->dev, addr: lsop->rspdma,
1426	size: sizeof(*lsop->rspbuf), dir: DMA_TO_DEVICE);
1427	fc_dma_unmap_single(dev: lport->dev, addr: lsop->rspdma,
1428	size: sizeof(*lsop->rspbuf), dir: DMA_TO_DEVICE);
1429
1430	kfree(objp: lsop->rspbuf);
1431	kfree(objp: lsop->rqstbuf);
1432	kfree(objp: lsop);
1433
1434	nvme_fc_rport_put(rport);
1435	}
1436
1437	static void
1438	nvme_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp)
1439	{
1440	struct nvmefc_ls_rcv_op *lsop = lsrsp->nvme_fc_private;
1441
1442	nvme_fc_xmt_ls_rsp_free(lsop);
1443	}
1444
1445	static void
1446	nvme_fc_xmt_ls_rsp(struct nvmefc_ls_rcv_op *lsop)
1447	{
1448	struct nvme_fc_rport *rport = lsop->rport;
1449	struct nvme_fc_lport *lport = rport->lport;
1450	struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1451	int ret;
1452
1453	fc_dma_sync_single_for_device(dev: lport->dev, addr: lsop->rspdma,
1454	size: sizeof(*lsop->rspbuf), dir: DMA_TO_DEVICE);
1455
1456	ret = lport->ops->xmt_ls_rsp(&lport->localport, &rport->remoteport,
1457	lsop->lsrsp);
1458	if (ret) {
1459	dev_warn(lport->dev,
1460	"LLDD rejected LS RSP xmt: LS %d status %d\n",
1461	w0->ls_cmd, ret);
1462	nvme_fc_xmt_ls_rsp_free(lsop);
1463	return;
1464	}
1465	}
1466
1467	static struct nvme_fc_ctrl *
1468	nvme_fc_match_disconn_ls(struct nvme_fc_rport *rport,
1469	struct nvmefc_ls_rcv_op *lsop)
1470	{
1471	struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1472	&lsop->rqstbuf->rq_dis_assoc;
1473	struct nvme_fc_ctrl *ctrl, *tmp, *ret = NULL;
1474	struct nvmefc_ls_rcv_op *oldls = NULL;
1475	u64 association_id = be64_to_cpu(rqst->associd.association_id);
1476	unsigned long flags;
1477
1478	spin_lock_irqsave(&rport->lock, flags);
1479
1480	list_for_each_entry_safe(ctrl, tmp, &rport->ctrl_list, ctrl_list) {
1481	if (!nvme_fc_ctrl_get(ctrl))
1482	continue;
1483	spin_lock(lock: &ctrl->lock);
1484	if (association_id == ctrl->association_id) {
1485	oldls = ctrl->rcv_disconn;
1486	ctrl->rcv_disconn = lsop;
1487	ret = ctrl;
1488	}
1489	spin_unlock(lock: &ctrl->lock);
1490	if (ret)
1491	/* leave the ctrl get reference */
1492	break;
1493	spin_unlock_irqrestore(lock: &rport->lock, flags);
1494	nvme_fc_ctrl_put(ctrl);
1495	spin_lock_irqsave(&rport->lock, flags);
1496	}
1497
1498	spin_unlock_irqrestore(lock: &rport->lock, flags);
1499
1500	/* transmit a response for anything that was pending */
1501	if (oldls) {
1502	dev_info(rport->lport->dev,
1503	"NVME-FC{%d}: Multiple Disconnect Association "
1504	"LS's received\n", ctrl->cnum);
1505	/* overwrite good response with bogus failure */
1506	oldls->lsrsp->rsplen = nvme_fc_format_rjt(buf: oldls->rspbuf,
1507	buflen: sizeof(*oldls->rspbuf),
1508	ls_cmd: rqst->w0.ls_cmd,
1509	reason: FCNVME_RJT_RC_UNAB,
1510	explanation: FCNVME_RJT_EXP_NONE, vendor: 0);
1511	nvme_fc_xmt_ls_rsp(lsop: oldls);
1512	}
1513
1514	return ret;
1515	}
1516
1517	/*
1518	* returns true to mean LS handled and ls_rsp can be sent
1519	* returns false to defer ls_rsp xmt (will be done as part of
1520	* association termination)
1521	*/
1522	static bool
1523	nvme_fc_ls_disconnect_assoc(struct nvmefc_ls_rcv_op *lsop)
1524	{
1525	struct nvme_fc_rport *rport = lsop->rport;
1526	struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1527	&lsop->rqstbuf->rq_dis_assoc;
1528	struct fcnvme_ls_disconnect_assoc_acc *acc =
1529	&lsop->rspbuf->rsp_dis_assoc;
1530	struct nvme_fc_ctrl *ctrl = NULL;
1531	int ret = 0;
1532
1533	memset(acc, 0, sizeof(*acc));
1534
1535	ret = nvmefc_vldt_lsreq_discon_assoc(rqstlen: lsop->rqstdatalen, rqst);
1536	if (!ret) {
1537	/* match an active association */
1538	ctrl = nvme_fc_match_disconn_ls(rport, lsop);
1539	if (!ctrl)
1540	ret = VERR_NO_ASSOC;
1541	}
1542
1543	if (ret) {
1544	dev_info(rport->lport->dev,
1545	"Disconnect LS failed: %s\n",
1546	validation_errors[ret]);
1547	lsop->lsrsp->rsplen = nvme_fc_format_rjt(buf: acc,
1548	buflen: sizeof(*acc), ls_cmd: rqst->w0.ls_cmd,
1549	reason: (ret == VERR_NO_ASSOC) ?
1550	FCNVME_RJT_RC_INV_ASSOC :
1551	FCNVME_RJT_RC_LOGIC,
1552	explanation: FCNVME_RJT_EXP_NONE, vendor: 0);
1553	return true;
1554	}
1555
1556	/* format an ACCept response */
1557
1558	lsop->lsrsp->rsplen = sizeof(*acc);
1559
1560	nvme_fc_format_rsp_hdr(buf: acc, ls_cmd: FCNVME_LS_ACC,
1561	desc_len: fcnvme_lsdesc_len(
1562	sz: sizeof(struct fcnvme_ls_disconnect_assoc_acc)),
1563	rqst_ls_cmd: FCNVME_LS_DISCONNECT_ASSOC);
1564
1565	/*
1566	* the transmit of the response will occur after the exchanges
1567	* for the association have been ABTS'd by
1568	* nvme_fc_delete_association().
1569	*/
1570
1571	/* fail the association */
1572	nvme_fc_error_recovery(ctrl, errmsg: "Disconnect Association LS received");
1573
1574	/* release the reference taken by nvme_fc_match_disconn_ls() */
1575	nvme_fc_ctrl_put(ctrl);
1576
1577	return false;
1578	}
1579
1580	/*
1581	* Actual Processing routine for received FC-NVME LS Requests from the LLD
1582	* returns true if a response should be sent afterward, false if rsp will
1583	* be sent asynchronously.
1584	*/
1585	static bool
1586	nvme_fc_handle_ls_rqst(struct nvmefc_ls_rcv_op *lsop)
1587	{
1588	struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1589	bool ret = true;
1590
1591	lsop->lsrsp->nvme_fc_private = lsop;
1592	lsop->lsrsp->rspbuf = lsop->rspbuf;
1593	lsop->lsrsp->rspdma = lsop->rspdma;
1594	lsop->lsrsp->done = nvme_fc_xmt_ls_rsp_done;
1595	/* Be preventative. handlers will later set to valid length */
1596	lsop->lsrsp->rsplen = 0;
1597
1598	/*
1599	* handlers:
1600	* parse request input, execute the request, and format the
1601	* LS response
1602	*/
1603	switch (w0->ls_cmd) {
1604	case FCNVME_LS_DISCONNECT_ASSOC:
1605	ret = nvme_fc_ls_disconnect_assoc(lsop);
1606	break;
1607	case FCNVME_LS_DISCONNECT_CONN:
1608	lsop->lsrsp->rsplen = nvme_fc_format_rjt(buf: lsop->rspbuf,
1609	buflen: sizeof(*lsop->rspbuf), ls_cmd: w0->ls_cmd,
1610	reason: FCNVME_RJT_RC_UNSUP, explanation: FCNVME_RJT_EXP_NONE, vendor: 0);
1611	break;
1612	case FCNVME_LS_CREATE_ASSOCIATION:
1613	case FCNVME_LS_CREATE_CONNECTION:
1614	lsop->lsrsp->rsplen = nvme_fc_format_rjt(buf: lsop->rspbuf,
1615	buflen: sizeof(*lsop->rspbuf), ls_cmd: w0->ls_cmd,
1616	reason: FCNVME_RJT_RC_LOGIC, explanation: FCNVME_RJT_EXP_NONE, vendor: 0);
1617	break;
1618	default:
1619	lsop->lsrsp->rsplen = nvme_fc_format_rjt(buf: lsop->rspbuf,
1620	buflen: sizeof(*lsop->rspbuf), ls_cmd: w0->ls_cmd,
1621	reason: FCNVME_RJT_RC_INVAL, explanation: FCNVME_RJT_EXP_NONE, vendor: 0);
1622	break;
1623	}
1624
1625	return(ret);
1626	}
1627
1628	static void
1629	nvme_fc_handle_ls_rqst_work(struct work_struct *work)
1630	{
1631	struct nvme_fc_rport *rport =
1632	container_of(work, struct nvme_fc_rport, lsrcv_work);
1633	struct fcnvme_ls_rqst_w0 *w0;
1634	struct nvmefc_ls_rcv_op *lsop;
1635	unsigned long flags;
1636	bool sendrsp;
1637
1638	restart:
1639	sendrsp = true;
1640	spin_lock_irqsave(&rport->lock, flags);
1641	list_for_each_entry(lsop, &rport->ls_rcv_list, lsrcv_list) {
1642	if (lsop->handled)
1643	continue;
1644
1645	lsop->handled = true;
1646	if (rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
1647	spin_unlock_irqrestore(lock: &rport->lock, flags);
1648	sendrsp = nvme_fc_handle_ls_rqst(lsop);
1649	} else {
1650	spin_unlock_irqrestore(lock: &rport->lock, flags);
1651	w0 = &lsop->rqstbuf->w0;
1652	lsop->lsrsp->rsplen = nvme_fc_format_rjt(
1653	buf: lsop->rspbuf,
1654	buflen: sizeof(*lsop->rspbuf),
1655	ls_cmd: w0->ls_cmd,
1656	reason: FCNVME_RJT_RC_UNAB,
1657	explanation: FCNVME_RJT_EXP_NONE, vendor: 0);
1658	}
1659	if (sendrsp)
1660	nvme_fc_xmt_ls_rsp(lsop);
1661	goto restart;
1662	}
1663	spin_unlock_irqrestore(lock: &rport->lock, flags);
1664	}
1665
1666	static
1667	void nvme_fc_rcv_ls_req_err_msg(struct nvme_fc_lport *lport,
1668	struct fcnvme_ls_rqst_w0 *w0)
1669	{
1670	dev_info(lport->dev, "RCV %s LS failed: No memory\n",
1671	(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1672	nvmefc_ls_names[w0->ls_cmd] : "");
1673	}
1674
1675	/**
1676	* nvme_fc_rcv_ls_req - transport entry point called by an LLDD
1677	* upon the reception of a NVME LS request.
1678	*
1679	* The nvme-fc layer will copy payload to an internal structure for
1680	* processing. As such, upon completion of the routine, the LLDD may
1681	* immediately free/reuse the LS request buffer passed in the call.
1682	*
1683	* If this routine returns error, the LLDD should abort the exchange.
1684	*
1685	* @portptr: pointer to the (registered) remote port that the LS
1686	* was received from. The remoteport is associated with
1687	* a specific localport.
1688	* @lsrsp: pointer to a nvmefc_ls_rsp response structure to be
1689	* used to reference the exchange corresponding to the LS
1690	* when issuing an ls response.
1691	* @lsreqbuf: pointer to the buffer containing the LS Request
1692	* @lsreqbuf_len: length, in bytes, of the received LS request
1693	*/
1694	int
1695	nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *portptr,
1696	struct nvmefc_ls_rsp *lsrsp,
1697	void *lsreqbuf, u32 lsreqbuf_len)
1698	{
1699	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
1700	struct nvme_fc_lport *lport = rport->lport;
1701	struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf;
1702	struct nvmefc_ls_rcv_op *lsop;
1703	unsigned long flags;
1704	int ret;
1705
1706	nvme_fc_rport_get(rport);
1707
1708	/* validate there's a routine to transmit a response */
1709	if (!lport->ops->xmt_ls_rsp) {
1710	dev_info(lport->dev,
1711	"RCV %s LS failed: no LLDD xmt_ls_rsp\n",
1712	(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1713	nvmefc_ls_names[w0->ls_cmd] : "");
1714	ret = -EINVAL;
1715	goto out_put;
1716	}
1717
1718	if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) {
1719	dev_info(lport->dev,
1720	"RCV %s LS failed: payload too large\n",
1721	(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1722	nvmefc_ls_names[w0->ls_cmd] : "");
1723	ret = -E2BIG;
1724	goto out_put;
1725	}
1726
1727	lsop = kzalloc(sizeof(*lsop), GFP_KERNEL);
1728	if (!lsop) {
1729	nvme_fc_rcv_ls_req_err_msg(lport, w0);
1730	ret = -ENOMEM;
1731	goto out_put;
1732	}
1733
1734	lsop->rqstbuf = kzalloc(sizeof(*lsop->rqstbuf), GFP_KERNEL);
1735	lsop->rspbuf = kzalloc(sizeof(*lsop->rspbuf), GFP_KERNEL);
1736	if (!lsop->rqstbuf || !lsop->rspbuf) {
1737	nvme_fc_rcv_ls_req_err_msg(lport, w0);
1738	ret = -ENOMEM;
1739	goto out_free;
1740	}
1741
1742	lsop->rspdma = fc_dma_map_single(dev: lport->dev, ptr: lsop->rspbuf,
1743	size: sizeof(*lsop->rspbuf),
1744	dir: DMA_TO_DEVICE);
1745	if (fc_dma_mapping_error(dev: lport->dev, dma_addr: lsop->rspdma)) {
1746	dev_info(lport->dev,
1747	"RCV %s LS failed: DMA mapping failure\n",
1748	(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1749	nvmefc_ls_names[w0->ls_cmd] : "");
1750	ret = -EFAULT;
1751	goto out_free;
1752	}
1753
1754	lsop->rport = rport;
1755	lsop->lsrsp = lsrsp;
1756
1757	memcpy(lsop->rqstbuf, lsreqbuf, lsreqbuf_len);
1758	lsop->rqstdatalen = lsreqbuf_len;
1759
1760	spin_lock_irqsave(&rport->lock, flags);
1761	if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE) {
1762	spin_unlock_irqrestore(lock: &rport->lock, flags);
1763	ret = -ENOTCONN;
1764	goto out_unmap;
1765	}
1766	list_add_tail(new: &lsop->lsrcv_list, head: &rport->ls_rcv_list);
1767	spin_unlock_irqrestore(lock: &rport->lock, flags);
1768
1769	schedule_work(work: &rport->lsrcv_work);
1770
1771	return 0;
1772
1773	out_unmap:
1774	fc_dma_unmap_single(dev: lport->dev, addr: lsop->rspdma,
1775	size: sizeof(*lsop->rspbuf), dir: DMA_TO_DEVICE);
1776	out_free:
1777	kfree(objp: lsop->rspbuf);
1778	kfree(objp: lsop->rqstbuf);
1779	kfree(objp: lsop);
1780	out_put:
1781	nvme_fc_rport_put(rport);
1782	return ret;
1783	}
1784	EXPORT_SYMBOL_GPL(nvme_fc_rcv_ls_req);
1785
1786
1787	/* *********************** NVME Ctrl Routines **************************** */
1788
1789	static void
1790	__nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl,
1791	struct nvme_fc_fcp_op *op)
1792	{
1793	fc_dma_unmap_single(dev: ctrl->lport->dev, addr: op->fcp_req.rspdma,
1794	size: sizeof(op->rsp_iu), dir: DMA_FROM_DEVICE);
1795	fc_dma_unmap_single(dev: ctrl->lport->dev, addr: op->fcp_req.cmddma,
1796	size: sizeof(op->cmd_iu), dir: DMA_TO_DEVICE);
1797
1798	atomic_set(v: &op->state, i: FCPOP_STATE_UNINIT);
1799	}
1800
1801	static void
1802	nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq,
1803	unsigned int hctx_idx)
1804	{
1805	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
1806
1807	return __nvme_fc_exit_request(ctrl: to_fc_ctrl(ctrl: set->driver_data), op);
1808	}
1809
1810	static int
1811	__nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op)
1812	{
1813	unsigned long flags;
1814	int opstate;
1815
1816	spin_lock_irqsave(&ctrl->lock, flags);
1817	opstate = atomic_xchg(v: &op->state, new: FCPOP_STATE_ABORTED);
1818	if (opstate != FCPOP_STATE_ACTIVE)
1819	atomic_set(v: &op->state, i: opstate);
1820	else if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) {
1821	op->flags |= FCOP_FLAGS_TERMIO;
1822	ctrl->iocnt++;
1823	}
1824	spin_unlock_irqrestore(lock: &ctrl->lock, flags);
1825
1826	if (opstate != FCPOP_STATE_ACTIVE)
1827	return -ECANCELED;
1828
1829	ctrl->lport->ops->fcp_abort(&ctrl->lport->localport,
1830	&ctrl->rport->remoteport,
1831	op->queue->lldd_handle,
1832	&op->fcp_req);
1833
1834	return 0;
1835	}
1836
1837	static void
1838	nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl)
1839	{
1840	struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops;
1841	int i;
1842
1843	/* ensure we've initialized the ops once */
1844	if (!(aen_op->flags & FCOP_FLAGS_AEN))
1845	return;
1846
1847	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++)
1848	__nvme_fc_abort_op(ctrl, op: aen_op);
1849	}
1850
1851	static inline void
1852	__nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
1853	struct nvme_fc_fcp_op *op, int opstate)
1854	{
1855	unsigned long flags;
1856
1857	if (opstate == FCPOP_STATE_ABORTED) {
1858	spin_lock_irqsave(&ctrl->lock, flags);
1859	if (test_bit(FCCTRL_TERMIO, &ctrl->flags) &&
1860	op->flags & FCOP_FLAGS_TERMIO) {
1861	if (!--ctrl->iocnt)
1862	wake_up(&ctrl->ioabort_wait);
1863	}
1864	spin_unlock_irqrestore(lock: &ctrl->lock, flags);
1865	}
1866	}
1867
1868	static void
1869	nvme_fc_ctrl_ioerr_work(struct work_struct *work)
1870	{
1871	struct nvme_fc_ctrl *ctrl =
1872	container_of(work, struct nvme_fc_ctrl, ioerr_work);
1873
1874	nvme_fc_error_recovery(ctrl, errmsg: "transport detected io error");
1875	}
1876
1877	/*
1878	* nvme_fc_io_getuuid - Routine called to get the appid field
1879	* associated with request by the lldd
1880	* @req:IO request from nvme fc to driver
1881	* Returns: UUID if there is an appid associated with VM or
1882	* NULL if the user/libvirt has not set the appid to VM
1883	*/
1884	char *nvme_fc_io_getuuid(struct nvmefc_fcp_req *req)
1885	{
1886	struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(fcpreq: req);
1887	struct request *rq = op->rq;
1888
1889	if (!IS_ENABLED(CONFIG_BLK_CGROUP_FC_APPID) || !rq || !rq->bio)
1890	return NULL;
1891	return blkcg_get_fc_appid(bio: rq->bio);
1892	}
1893	EXPORT_SYMBOL_GPL(nvme_fc_io_getuuid);
1894
1895	static void
1896	nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
1897	{
1898	struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(fcpreq: req);
1899	struct request *rq = op->rq;
1900	struct nvmefc_fcp_req *freq = &op->fcp_req;
1901	struct nvme_fc_ctrl *ctrl = op->ctrl;
1902	struct nvme_fc_queue *queue = op->queue;
1903	struct nvme_completion *cqe = &op->rsp_iu.cqe;
1904	struct nvme_command *sqe = &op->cmd_iu.sqe;
1905	__le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1);
1906	union nvme_result result;
1907	bool terminate_assoc = true;
1908	int opstate;
1909
1910	/*
1911	* WARNING:
1912	* The current linux implementation of a nvme controller
1913	* allocates a single tag set for all io queues and sizes
1914	* the io queues to fully hold all possible tags. Thus, the
1915	* implementation does not reference or care about the sqhd
1916	* value as it never needs to use the sqhd/sqtail pointers
1917	* for submission pacing.
1918	*
1919	* This affects the FC-NVME implementation in two ways:
1920	* 1) As the value doesn't matter, we don't need to waste
1921	* cycles extracting it from ERSPs and stamping it in the
1922	* cases where the transport fabricates CQEs on successful
1923	* completions.
1924	* 2) The FC-NVME implementation requires that delivery of
1925	* ERSP completions are to go back to the nvme layer in order
1926	* relative to the rsn, such that the sqhd value will always
1927	* be "in order" for the nvme layer. As the nvme layer in
1928	* linux doesn't care about sqhd, there's no need to return
1929	* them in order.
1930	*
1931	* Additionally:
1932	* As the core nvme layer in linux currently does not look at
1933	* every field in the cqe - in cases where the FC transport must
1934	* fabricate a CQE, the following fields will not be set as they
1935	* are not referenced:
1936	* cqe.sqid, cqe.sqhd, cqe.command_id
1937	*
1938	* Failure or error of an individual i/o, in a transport
1939	* detected fashion unrelated to the nvme completion status,
1940	* potentially cause the initiator and target sides to get out
1941	* of sync on SQ head/tail (aka outstanding io count allowed).
1942	* Per FC-NVME spec, failure of an individual command requires
1943	* the connection to be terminated, which in turn requires the
1944	* association to be terminated.
1945	*/
1946
1947	opstate = atomic_xchg(v: &op->state, new: FCPOP_STATE_COMPLETE);
1948
1949	fc_dma_sync_single_for_cpu(dev: ctrl->lport->dev, addr: op->fcp_req.rspdma,
1950	size: sizeof(op->rsp_iu), dir: DMA_FROM_DEVICE);
1951
1952	if (opstate == FCPOP_STATE_ABORTED)
1953	status = cpu_to_le16(NVME_SC_HOST_ABORTED_CMD << 1);
1954	else if (freq->status) {
1955	status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
1956	dev_info(ctrl->ctrl.device,
1957	"NVME-FC{%d}: io failed due to lldd error %d\n",
1958	ctrl->cnum, freq->status);
1959	}
1960
1961	/*
1962	* For the linux implementation, if we have an unsuccessful
1963	* status, the blk-mq layer can typically be called with the
1964	* non-zero status and the content of the cqe isn't important.
1965	*/
1966	if (status)
1967	goto done;
1968
1969	/*
1970	* command completed successfully relative to the wire
1971	* protocol. However, validate anything received and
1972	* extract the status and result from the cqe (create it
1973	* where necessary).
1974	*/
1975
1976	switch (freq->rcv_rsplen) {
1977
1978	case 0:
1979	case NVME_FC_SIZEOF_ZEROS_RSP:
1980	/*
1981	* No response payload or 12 bytes of payload (which
1982	* should all be zeros) are considered successful and
1983	* no payload in the CQE by the transport.
1984	*/
1985	if (freq->transferred_length !=
1986	be32_to_cpu(op->cmd_iu.data_len)) {
1987	status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
1988	dev_info(ctrl->ctrl.device,
1989	"NVME-FC{%d}: io failed due to bad transfer "
1990	"length: %d vs expected %d\n",
1991	ctrl->cnum, freq->transferred_length,
1992	be32_to_cpu(op->cmd_iu.data_len));
1993	goto done;
1994	}
1995	result.u64 = 0;
1996	break;
1997
1998	case sizeof(struct nvme_fc_ersp_iu):
1999	/*
2000	* The ERSP IU contains a full completion with CQE.
2001	* Validate ERSP IU and look at cqe.
2002	*/
2003	if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) !=
2004	(freq->rcv_rsplen / 4) ||
2005	be32_to_cpu(op->rsp_iu.xfrd_len) !=
2006	freq->transferred_length ||
2007	op->rsp_iu.ersp_result ||
2008	sqe->common.command_id != cqe->command_id)) {
2009	status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2010	dev_info(ctrl->ctrl.device,
2011	"NVME-FC{%d}: io failed due to bad NVMe_ERSP: "
2012	"iu len %d, xfr len %d vs %d, status code "
2013	"%d, cmdid %d vs %d\n",
2014	ctrl->cnum, be16_to_cpu(op->rsp_iu.iu_len),
2015	be32_to_cpu(op->rsp_iu.xfrd_len),
2016	freq->transferred_length,
2017	op->rsp_iu.ersp_result,
2018	sqe->common.command_id,
2019	cqe->command_id);
2020	goto done;
2021	}
2022	result = cqe->result;
2023	status = cqe->status;
2024	break;
2025
2026	default:
2027	status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2028	dev_info(ctrl->ctrl.device,
2029	"NVME-FC{%d}: io failed due to odd NVMe_xRSP iu "
2030	"len %d\n",
2031	ctrl->cnum, freq->rcv_rsplen);
2032	goto done;
2033	}
2034
2035	terminate_assoc = false;
2036
2037	done:
2038	if (op->flags & FCOP_FLAGS_AEN) {
2039	nvme_complete_async_event(ctrl: &queue->ctrl->ctrl, status, res: &result);
2040	__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2041	atomic_set(v: &op->state, i: FCPOP_STATE_IDLE);
2042	op->flags = FCOP_FLAGS_AEN; /* clear other flags */
2043	nvme_fc_ctrl_put(ctrl);
2044	goto check_error;
2045	}
2046
2047	__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2048	if (!nvme_try_complete_req(req: rq, status, result))
2049	nvme_fc_complete_rq(rq);
2050
2051	check_error:
2052	if (terminate_assoc &&
2053	nvme_ctrl_state(ctrl: &ctrl->ctrl) != NVME_CTRL_RESETTING)
2054	queue_work(wq: nvme_reset_wq, work: &ctrl->ioerr_work);
2055	}
2056
2057	static int
2058	__nvme_fc_init_request(struct nvme_fc_ctrl *ctrl,
2059	struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op,
2060	struct request *rq, u32 rqno)
2061	{
2062	struct nvme_fcp_op_w_sgl *op_w_sgl =
2063	container_of(op, typeof(*op_w_sgl), op);
2064	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2065	int ret = 0;
2066
2067	memset(op, 0, sizeof(*op));
2068	op->fcp_req.cmdaddr = &op->cmd_iu;
2069	op->fcp_req.cmdlen = sizeof(op->cmd_iu);
2070	op->fcp_req.rspaddr = &op->rsp_iu;
2071	op->fcp_req.rsplen = sizeof(op->rsp_iu);
2072	op->fcp_req.done = nvme_fc_fcpio_done;
2073	op->ctrl = ctrl;
2074	op->queue = queue;
2075	op->rq = rq;
2076	op->rqno = rqno;
2077
2078	cmdiu->format_id = NVME_CMD_FORMAT_ID;
2079	cmdiu->fc_id = NVME_CMD_FC_ID;
2080	cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));
2081	if (queue->qnum)
2082	cmdiu->rsv_cat = fccmnd_set_cat_css(rsv_cat: 0,
2083	css: (NVME_CC_CSS_NVM >> NVME_CC_CSS_SHIFT));
2084	else
2085	cmdiu->rsv_cat = fccmnd_set_cat_admin(rsv_cat: 0);
2086
2087	op->fcp_req.cmddma = fc_dma_map_single(dev: ctrl->lport->dev,
2088	ptr: &op->cmd_iu, size: sizeof(op->cmd_iu), dir: DMA_TO_DEVICE);
2089	if (fc_dma_mapping_error(dev: ctrl->lport->dev, dma_addr: op->fcp_req.cmddma)) {
2090	dev_err(ctrl->dev,
2091	"FCP Op failed - cmdiu dma mapping failed.\n");
2092	ret = -EFAULT;
2093	goto out_on_error;
2094	}
2095
2096	op->fcp_req.rspdma = fc_dma_map_single(dev: ctrl->lport->dev,
2097	ptr: &op->rsp_iu, size: sizeof(op->rsp_iu),
2098	dir: DMA_FROM_DEVICE);
2099	if (fc_dma_mapping_error(dev: ctrl->lport->dev, dma_addr: op->fcp_req.rspdma)) {
2100	dev_err(ctrl->dev,
2101	"FCP Op failed - rspiu dma mapping failed.\n");
2102	ret = -EFAULT;
2103	}
2104
2105	atomic_set(v: &op->state, i: FCPOP_STATE_IDLE);
2106	out_on_error:
2107	return ret;
2108	}
2109
2110	static int
2111	nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq,
2112	unsigned int hctx_idx, unsigned int numa_node)
2113	{
2114	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: set->driver_data);
2115	struct nvme_fcp_op_w_sgl *op = blk_mq_rq_to_pdu(rq);
2116	int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
2117	struct nvme_fc_queue *queue = &ctrl->queues[queue_idx];
2118	int res;
2119
2120	res = __nvme_fc_init_request(ctrl, queue, op: &op->op, rq, rqno: queue->rqcnt++);
2121	if (res)
2122	return res;
2123	op->op.fcp_req.first_sgl = op->sgl;
2124	op->op.fcp_req.private = &op->priv[0];
2125	nvme_req(req: rq)->ctrl = &ctrl->ctrl;
2126	nvme_req(req: rq)->cmd = &op->op.cmd_iu.sqe;
2127	return res;
2128	}
2129
2130	static int
2131	nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl)
2132	{
2133	struct nvme_fc_fcp_op *aen_op;
2134	struct nvme_fc_cmd_iu *cmdiu;
2135	struct nvme_command *sqe;
2136	void *private = NULL;
2137	int i, ret;
2138
2139	aen_op = ctrl->aen_ops;
2140	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2141	if (ctrl->lport->ops->fcprqst_priv_sz) {
2142	private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz,
2143	GFP_KERNEL);
2144	if (!private)
2145	return -ENOMEM;
2146	}
2147
2148	cmdiu = &aen_op->cmd_iu;
2149	sqe = &cmdiu->sqe;
2150	ret = __nvme_fc_init_request(ctrl, queue: &ctrl->queues[0],
2151	op: aen_op, rq: (struct request *)NULL,
2152	rqno: (NVME_AQ_BLK_MQ_DEPTH + i));
2153	if (ret) {
2154	kfree(objp: private);
2155	return ret;
2156	}
2157
2158	aen_op->flags = FCOP_FLAGS_AEN;
2159	aen_op->fcp_req.private = private;
2160
2161	memset(sqe, 0, sizeof(*sqe));
2162	sqe->common.opcode = nvme_admin_async_event;
2163	/* Note: core layer may overwrite the sqe.command_id value */
2164	sqe->common.command_id = NVME_AQ_BLK_MQ_DEPTH + i;
2165	}
2166	return 0;
2167	}
2168
2169	static void
2170	nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl)
2171	{
2172	struct nvme_fc_fcp_op *aen_op;
2173	int i;
2174
2175	cancel_work_sync(work: &ctrl->ctrl.async_event_work);
2176	aen_op = ctrl->aen_ops;
2177	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2178	__nvme_fc_exit_request(ctrl, op: aen_op);
2179
2180	kfree(objp: aen_op->fcp_req.private);
2181	aen_op->fcp_req.private = NULL;
2182	}
2183	}
2184
2185	static inline int
2186	__nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int qidx)
2187	{
2188	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: data);
2189	struct nvme_fc_queue *queue = &ctrl->queues[qidx];
2190
2191	hctx->driver_data = queue;
2192	queue->hctx = hctx;
2193	return 0;
2194	}
2195
2196	static int
2197	nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx)
2198	{
2199	return __nvme_fc_init_hctx(hctx, data, qidx: hctx_idx + 1);
2200	}
2201
2202	static int
2203	nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
2204	unsigned int hctx_idx)
2205	{
2206	return __nvme_fc_init_hctx(hctx, data, qidx: hctx_idx);
2207	}
2208
2209	static void
2210	nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx)
2211	{
2212	struct nvme_fc_queue *queue;
2213
2214	queue = &ctrl->queues[idx];
2215	memset(queue, 0, sizeof(*queue));
2216	queue->ctrl = ctrl;
2217	queue->qnum = idx;
2218	atomic_set(v: &queue->csn, i: 0);
2219	queue->dev = ctrl->dev;
2220
2221	if (idx > 0)
2222	queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
2223	else
2224	queue->cmnd_capsule_len = sizeof(struct nvme_command);
2225
2226	/*
2227	* Considered whether we should allocate buffers for all SQEs
2228	* and CQEs and dma map them - mapping their respective entries
2229	* into the request structures (kernel vm addr and dma address)
2230	* thus the driver could use the buffers/mappings directly.
2231	* It only makes sense if the LLDD would use them for its
2232	* messaging api. It's very unlikely most adapter api's would use
2233	* a native NVME sqe/cqe. More reasonable if FC-NVME IU payload
2234	* structures were used instead.
2235	*/
2236	}
2237
2238	/*
2239	* This routine terminates a queue at the transport level.
2240	* The transport has already ensured that all outstanding ios on
2241	* the queue have been terminated.
2242	* The transport will send a Disconnect LS request to terminate
2243	* the queue's connection. Termination of the admin queue will also
2244	* terminate the association at the target.
2245	*/
2246	static void
2247	nvme_fc_free_queue(struct nvme_fc_queue *queue)
2248	{
2249	if (!test_and_clear_bit(nr: NVME_FC_Q_CONNECTED, addr: &queue->flags))
2250	return;
2251
2252	clear_bit(nr: NVME_FC_Q_LIVE, addr: &queue->flags);
2253	/*
2254	* Current implementation never disconnects a single queue.
2255	* It always terminates a whole association. So there is never
2256	* a disconnect(queue) LS sent to the target.
2257	*/
2258
2259	queue->connection_id = 0;
2260	atomic_set(v: &queue->csn, i: 0);
2261	}
2262
2263	static void
2264	__nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl,
2265	struct nvme_fc_queue *queue, unsigned int qidx)
2266	{
2267	if (ctrl->lport->ops->delete_queue)
2268	ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx,
2269	queue->lldd_handle);
2270	queue->lldd_handle = NULL;
2271	}
2272
2273	static void
2274	nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl)
2275	{
2276	int i;
2277
2278	for (i = 1; i < ctrl->ctrl.queue_count; i++)
2279	nvme_fc_free_queue(queue: &ctrl->queues[i]);
2280	}
2281
2282	static int
2283	__nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl,
2284	struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize)
2285	{
2286	int ret = 0;
2287
2288	queue->lldd_handle = NULL;
2289	if (ctrl->lport->ops->create_queue)
2290	ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport,
2291	qidx, qsize, &queue->lldd_handle);
2292
2293	return ret;
2294	}
2295
2296	static void
2297	nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl)
2298	{
2299	struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1];
2300	int i;
2301
2302	for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--)
2303	__nvme_fc_delete_hw_queue(ctrl, queue, qidx: i);
2304	}
2305
2306	static int
2307	nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2308	{
2309	struct nvme_fc_queue *queue = &ctrl->queues[1];
2310	int i, ret;
2311
2312	for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) {
2313	ret = __nvme_fc_create_hw_queue(ctrl, queue, qidx: i, qsize);
2314	if (ret)
2315	goto delete_queues;
2316	}
2317
2318	return 0;
2319
2320	delete_queues:
2321	for (; i > 0; i--)
2322	__nvme_fc_delete_hw_queue(ctrl, queue: &ctrl->queues[i], qidx: i);
2323	return ret;
2324	}
2325
2326	static int
2327	nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2328	{
2329	int i, ret = 0;
2330
2331	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
2332	ret = nvme_fc_connect_queue(ctrl, queue: &ctrl->queues[i], qsize,
2333	ersp_ratio: (qsize / 5));
2334	if (ret)
2335	break;
2336	ret = nvmf_connect_io_queue(ctrl: &ctrl->ctrl, qid: i);
2337	if (ret)
2338	break;
2339
2340	set_bit(nr: NVME_FC_Q_LIVE, addr: &ctrl->queues[i].flags);
2341	}
2342
2343	return ret;
2344	}
2345
2346	static void
2347	nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl)
2348	{
2349	int i;
2350
2351	for (i = 1; i < ctrl->ctrl.queue_count; i++)
2352	nvme_fc_init_queue(ctrl, idx: i);
2353	}
2354
2355	static void
2356	nvme_fc_ctrl_free(struct kref *ref)
2357	{
2358	struct nvme_fc_ctrl *ctrl =
2359	container_of(ref, struct nvme_fc_ctrl, ref);
2360	unsigned long flags;
2361
2362	/* remove from rport list */
2363	spin_lock_irqsave(&ctrl->rport->lock, flags);
2364	list_del(entry: &ctrl->ctrl_list);
2365	spin_unlock_irqrestore(lock: &ctrl->rport->lock, flags);
2366
2367	kfree(objp: ctrl->queues);
2368
2369	put_device(dev: ctrl->dev);
2370	nvme_fc_rport_put(rport: ctrl->rport);
2371
2372	ida_free(&nvme_fc_ctrl_cnt, id: ctrl->cnum);
2373	if (ctrl->ctrl.opts)
2374	nvmf_free_options(opts: ctrl->ctrl.opts);
2375	kfree(objp: ctrl);
2376	}
2377
2378	static void
2379	nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl)
2380	{
2381	kref_put(kref: &ctrl->ref, release: nvme_fc_ctrl_free);
2382	}
2383
2384	static int
2385	nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl)
2386	{
2387	return kref_get_unless_zero(kref: &ctrl->ref);
2388	}
2389
2390	/*
2391	* All accesses from nvme core layer done - can now free the
2392	* controller. Called after last nvme_put_ctrl() call
2393	*/
2394	static void
2395	nvme_fc_free_ctrl(struct nvme_ctrl *nctrl)
2396	{
2397	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: nctrl);
2398
2399	WARN_ON(nctrl != &ctrl->ctrl);
2400
2401	nvme_fc_ctrl_put(ctrl);
2402	}
2403
2404	/*
2405	* This routine is used by the transport when it needs to find active
2406	* io on a queue that is to be terminated. The transport uses
2407	* blk_mq_tagset_busy_itr() to find the busy requests, which then invoke
2408	* this routine to kill them on a 1 by 1 basis.
2409	*
2410	* As FC allocates FC exchange for each io, the transport must contact
2411	* the LLDD to terminate the exchange, thus releasing the FC exchange.
2412	* After terminating the exchange the LLDD will call the transport's
2413	* normal io done path for the request, but it will have an aborted
2414	* status. The done path will return the io request back to the block
2415	* layer with an error status.
2416	*/
2417	static bool nvme_fc_terminate_exchange(struct request *req, void *data)
2418	{
2419	struct nvme_ctrl *nctrl = data;
2420	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: nctrl);
2421	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq: req);
2422
2423	op->nreq.flags |= NVME_REQ_CANCELLED;
2424	__nvme_fc_abort_op(ctrl, op);
2425	return true;
2426	}
2427
2428	/*
2429	* This routine runs through all outstanding commands on the association
2430	* and aborts them. This routine is typically called by the
2431	* delete_association routine. It is also called due to an error during
2432	* reconnect. In that scenario, it is most likely a command that initializes
2433	* the controller, including fabric Connect commands on io queues, that
2434	* may have timed out or failed thus the io must be killed for the connect
2435	* thread to see the error.
2436	*/
2437	static void
2438	__nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)
2439	{
2440	int q;
2441
2442	/*
2443	* if aborting io, the queues are no longer good, mark them
2444	* all as not live.
2445	*/
2446	if (ctrl->ctrl.queue_count > 1) {
2447	for (q = 1; q < ctrl->ctrl.queue_count; q++)
2448	clear_bit(nr: NVME_FC_Q_LIVE, addr: &ctrl->queues[q].flags);
2449	}
2450	clear_bit(nr: NVME_FC_Q_LIVE, addr: &ctrl->queues[0].flags);
2451
2452	/*
2453	* If io queues are present, stop them and terminate all outstanding
2454	* ios on them. As FC allocates FC exchange for each io, the
2455	* transport must contact the LLDD to terminate the exchange,
2456	* thus releasing the FC exchange. We use blk_mq_tagset_busy_itr()
2457	* to tell us what io's are busy and invoke a transport routine
2458	* to kill them with the LLDD. After terminating the exchange
2459	* the LLDD will call the transport's normal io done path, but it
2460	* will have an aborted status. The done path will return the
2461	* io requests back to the block layer as part of normal completions
2462	* (but with error status).
2463	*/
2464	if (ctrl->ctrl.queue_count > 1) {
2465	nvme_quiesce_io_queues(ctrl: &ctrl->ctrl);
2466	nvme_sync_io_queues(ctrl: &ctrl->ctrl);
2467	blk_mq_tagset_busy_iter(tagset: &ctrl->tag_set,
2468	fn: nvme_fc_terminate_exchange, priv: &ctrl->ctrl);
2469	blk_mq_tagset_wait_completed_request(tagset: &ctrl->tag_set);
2470	if (start_queues)
2471	nvme_unquiesce_io_queues(ctrl: &ctrl->ctrl);
2472	}
2473
2474	/*
2475	* Other transports, which don't have link-level contexts bound
2476	* to sqe's, would try to gracefully shutdown the controller by
2477	* writing the registers for shutdown and polling (call
2478	* nvme_disable_ctrl()). Given a bunch of i/o was potentially
2479	* just aborted and we will wait on those contexts, and given
2480	* there was no indication of how live the controller is on the
2481	* link, don't send more io to create more contexts for the
2482	* shutdown. Let the controller fail via keepalive failure if
2483	* its still present.
2484	*/
2485
2486	/*
2487	* clean up the admin queue. Same thing as above.
2488	*/
2489	nvme_quiesce_admin_queue(ctrl: &ctrl->ctrl);
2490	blk_sync_queue(q: ctrl->ctrl.admin_q);
2491	blk_mq_tagset_busy_iter(tagset: &ctrl->admin_tag_set,
2492	fn: nvme_fc_terminate_exchange, priv: &ctrl->ctrl);
2493	blk_mq_tagset_wait_completed_request(tagset: &ctrl->admin_tag_set);
2494	if (start_queues)
2495	nvme_unquiesce_admin_queue(ctrl: &ctrl->ctrl);
2496	}
2497
2498	static void
2499	nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg)
2500	{
2501	enum nvme_ctrl_state state = nvme_ctrl_state(ctrl: &ctrl->ctrl);
2502
2503	/*
2504	* if an error (io timeout, etc) while (re)connecting, the remote
2505	* port requested terminating of the association (disconnect_ls)
2506	* or an error (timeout or abort) occurred on an io while creating
2507	* the controller. Abort any ios on the association and let the
2508	* create_association error path resolve things.
2509	*/
2510	if (state == NVME_CTRL_CONNECTING) {
2511	__nvme_fc_abort_outstanding_ios(ctrl, start_queues: true);
2512	dev_warn(ctrl->ctrl.device,
2513	"NVME-FC{%d}: transport error during (re)connect\n",
2514	ctrl->cnum);
2515	return;
2516	}
2517
2518	/* Otherwise, only proceed if in LIVE state - e.g. on first error */
2519	if (state != NVME_CTRL_LIVE)
2520	return;
2521
2522	dev_warn(ctrl->ctrl.device,
2523	"NVME-FC{%d}: transport association event: %s\n",
2524	ctrl->cnum, errmsg);
2525	dev_warn(ctrl->ctrl.device,
2526	"NVME-FC{%d}: resetting controller\n", ctrl->cnum);
2527
2528	nvme_reset_ctrl(ctrl: &ctrl->ctrl);
2529	}
2530
2531	static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq)
2532	{
2533	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2534	struct nvme_fc_ctrl *ctrl = op->ctrl;
2535	u16 qnum = op->queue->qnum;
2536	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2537	struct nvme_command *sqe = &cmdiu->sqe;
2538
2539	/*
2540	* Attempt to abort the offending command. Command completion
2541	* will detect the aborted io and will fail the connection.
2542	*/
2543	dev_info(ctrl->ctrl.device,
2544	"NVME-FC{%d.%d}: io timeout: opcode %d fctype %d (%s) w10/11: "
2545	"x%08x/x%08x\n",
2546	ctrl->cnum, qnum, sqe->common.opcode, sqe->fabrics.fctype,
2547	nvme_fabrics_opcode_str(qnum, sqe),
2548	sqe->common.cdw10, sqe->common.cdw11);
2549	if (__nvme_fc_abort_op(ctrl, op))
2550	nvme_fc_error_recovery(ctrl, errmsg: "io timeout abort failed");
2551
2552	/*
2553	* the io abort has been initiated. Have the reset timer
2554	* restarted and the abort completion will complete the io
2555	* shortly. Avoids a synchronous wait while the abort finishes.
2556	*/
2557	return BLK_EH_RESET_TIMER;
2558	}
2559
2560	static int
2561	nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2562	struct nvme_fc_fcp_op *op)
2563	{
2564	struct nvmefc_fcp_req *freq = &op->fcp_req;
2565	int ret;
2566
2567	freq->sg_cnt = 0;
2568
2569	if (!blk_rq_nr_phys_segments(rq))
2570	return 0;
2571
2572	freq->sg_table.sgl = freq->first_sgl;
2573	ret = sg_alloc_table_chained(table: &freq->sg_table,
2574	nents: blk_rq_nr_phys_segments(rq), first_chunk: freq->sg_table.sgl,
2575	NVME_INLINE_SG_CNT);
2576	if (ret)
2577	return -ENOMEM;
2578
2579	op->nents = blk_rq_map_sg(rq, sglist: freq->sg_table.sgl);
2580	WARN_ON(op->nents > blk_rq_nr_phys_segments(rq));
2581	freq->sg_cnt = fc_dma_map_sg(dev: ctrl->lport->dev, sg: freq->sg_table.sgl,
2582	nents: op->nents, rq_dma_dir(rq));
2583	if (unlikely(freq->sg_cnt <= 0)) {
2584	sg_free_table_chained(table: &freq->sg_table, NVME_INLINE_SG_CNT);
2585	freq->sg_cnt = 0;
2586	return -EFAULT;
2587	}
2588
2589	/*
2590	* TODO: blk_integrity_rq(rq) for DIF
2591	*/
2592	return 0;
2593	}
2594
2595	static void
2596	nvme_fc_unmap_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2597	struct nvme_fc_fcp_op *op)
2598	{
2599	struct nvmefc_fcp_req *freq = &op->fcp_req;
2600
2601	if (!freq->sg_cnt)
2602	return;
2603
2604	fc_dma_unmap_sg(dev: ctrl->lport->dev, sg: freq->sg_table.sgl, nents: op->nents,
2605	rq_dma_dir(rq));
2606
2607	sg_free_table_chained(table: &freq->sg_table, NVME_INLINE_SG_CNT);
2608
2609	freq->sg_cnt = 0;
2610	}
2611
2612	/*
2613	* In FC, the queue is a logical thing. At transport connect, the target
2614	* creates its "queue" and returns a handle that is to be given to the
2615	* target whenever it posts something to the corresponding SQ. When an
2616	* SQE is sent on a SQ, FC effectively considers the SQE, or rather the
2617	* command contained within the SQE, an io, and assigns a FC exchange
2618	* to it. The SQE and the associated SQ handle are sent in the initial
2619	* CMD IU sents on the exchange. All transfers relative to the io occur
2620	* as part of the exchange. The CQE is the last thing for the io,
2621	* which is transferred (explicitly or implicitly) with the RSP IU
2622	* sent on the exchange. After the CQE is received, the FC exchange is
2623	* terminated and the Exchange may be used on a different io.
2624	*
2625	* The transport to LLDD api has the transport making a request for a
2626	* new fcp io request to the LLDD. The LLDD then allocates a FC exchange
2627	* resource and transfers the command. The LLDD will then process all
2628	* steps to complete the io. Upon completion, the transport done routine
2629	* is called.
2630	*
2631	* So - while the operation is outstanding to the LLDD, there is a link
2632	* level FC exchange resource that is also outstanding. This must be
2633	* considered in all cleanup operations.
2634	*/
2635	static blk_status_t
2636	nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
2637	struct nvme_fc_fcp_op *op, u32 data_len,
2638	enum nvmefc_fcp_datadir io_dir)
2639	{
2640	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2641	struct nvme_command *sqe = &cmdiu->sqe;
2642	int ret, opstate;
2643
2644	/*
2645	* before attempting to send the io, check to see if we believe
2646	* the target device is present
2647	*/
2648	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
2649	return BLK_STS_RESOURCE;
2650
2651	if (!nvme_fc_ctrl_get(ctrl))
2652	return BLK_STS_IOERR;
2653
2654	/* format the FC-NVME CMD IU and fcp_req */
2655	cmdiu->connection_id = cpu_to_be64(queue->connection_id);
2656	cmdiu->data_len = cpu_to_be32(data_len);
2657	switch (io_dir) {
2658	case NVMEFC_FCP_WRITE:
2659	cmdiu->flags = FCNVME_CMD_FLAGS_WRITE;
2660	break;
2661	case NVMEFC_FCP_READ:
2662	cmdiu->flags = FCNVME_CMD_FLAGS_READ;
2663	break;
2664	case NVMEFC_FCP_NODATA:
2665	cmdiu->flags = 0;
2666	break;
2667	}
2668	op->fcp_req.payload_length = data_len;
2669	op->fcp_req.io_dir = io_dir;
2670	op->fcp_req.transferred_length = 0;
2671	op->fcp_req.rcv_rsplen = 0;
2672	op->fcp_req.status = NVME_SC_SUCCESS;
2673	op->fcp_req.sqid = cpu_to_le16(queue->qnum);
2674
2675	/*
2676	* validate per fabric rules, set fields mandated by fabric spec
2677	* as well as those by FC-NVME spec.
2678	*/
2679	WARN_ON_ONCE(sqe->common.metadata);
2680	sqe->common.flags |= NVME_CMD_SGL_METABUF;
2681
2682	/*
2683	* format SQE DPTR field per FC-NVME rules:
2684	* type=0x5 Transport SGL Data Block Descriptor
2685	* subtype=0xA Transport-specific value
2686	* address=0
2687	* length=length of the data series
2688	*/
2689	sqe->rw.dptr.sgl.type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2690	NVME_SGL_FMT_TRANSPORT_A;
2691	sqe->rw.dptr.sgl.length = cpu_to_le32(data_len);
2692	sqe->rw.dptr.sgl.addr = 0;
2693
2694	if (!(op->flags & FCOP_FLAGS_AEN)) {
2695	ret = nvme_fc_map_data(ctrl, rq: op->rq, op);
2696	if (ret < 0) {
2697	nvme_cleanup_cmd(req: op->rq);
2698	nvme_fc_ctrl_put(ctrl);
2699	if (ret == -ENOMEM || ret == -EAGAIN)
2700	return BLK_STS_RESOURCE;
2701	return BLK_STS_IOERR;
2702	}
2703	}
2704
2705	fc_dma_sync_single_for_device(dev: ctrl->lport->dev, addr: op->fcp_req.cmddma,
2706	size: sizeof(op->cmd_iu), dir: DMA_TO_DEVICE);
2707
2708	atomic_set(v: &op->state, i: FCPOP_STATE_ACTIVE);
2709
2710	if (!(op->flags & FCOP_FLAGS_AEN))
2711	nvme_start_request(rq: op->rq);
2712
2713	cmdiu->csn = cpu_to_be32(atomic_inc_return(&queue->csn));
2714	ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport,
2715	&ctrl->rport->remoteport,
2716	queue->lldd_handle, &op->fcp_req);
2717
2718	if (ret) {
2719	/*
2720	* If the lld fails to send the command is there an issue with
2721	* the csn value? If the command that fails is the Connect,
2722	* no - as the connection won't be live. If it is a command
2723	* post-connect, it's possible a gap in csn may be created.
2724	* Does this matter? As Linux initiators don't send fused
2725	* commands, no. The gap would exist, but as there's nothing
2726	* that depends on csn order to be delivered on the target
2727	* side, it shouldn't hurt. It would be difficult for a
2728	* target to even detect the csn gap as it has no idea when the
2729	* cmd with the csn was supposed to arrive.
2730	*/
2731	opstate = atomic_xchg(v: &op->state, new: FCPOP_STATE_COMPLETE);
2732	__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2733
2734	if (!(op->flags & FCOP_FLAGS_AEN)) {
2735	nvme_fc_unmap_data(ctrl, rq: op->rq, op);
2736	nvme_cleanup_cmd(req: op->rq);
2737	}
2738
2739	nvme_fc_ctrl_put(ctrl);
2740
2741	if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE &&
2742	ret != -EBUSY)
2743	return BLK_STS_IOERR;
2744
2745	return BLK_STS_RESOURCE;
2746	}
2747
2748	return BLK_STS_OK;
2749	}
2750
2751	static blk_status_t
2752	nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx,
2753	const struct blk_mq_queue_data *bd)
2754	{
2755	struct nvme_ns *ns = hctx->queue->queuedata;
2756	struct nvme_fc_queue *queue = hctx->driver_data;
2757	struct nvme_fc_ctrl *ctrl = queue->ctrl;
2758	struct request *rq = bd->rq;
2759	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2760	enum nvmefc_fcp_datadir io_dir;
2761	bool queue_ready = test_bit(NVME_FC_Q_LIVE, &queue->flags);
2762	u32 data_len;
2763	blk_status_t ret;
2764
2765	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE ||
2766	!nvme_check_ready(ctrl: &queue->ctrl->ctrl, rq, queue_live: queue_ready))
2767	return nvme_fail_nonready_command(ctrl: &queue->ctrl->ctrl, req: rq);
2768
2769	ret = nvme_setup_cmd(ns, req: rq);
2770	if (ret)
2771	return ret;
2772
2773	/*
2774	* nvme core doesn't quite treat the rq opaquely. Commands such
2775	* as WRITE ZEROES will return a non-zero rq payload_bytes yet
2776	* there is no actual payload to be transferred.
2777	* To get it right, key data transmission on there being 1 or
2778	* more physical segments in the sg list. If there are no
2779	* physical segments, there is no payload.
2780	*/
2781	if (blk_rq_nr_phys_segments(rq)) {
2782	data_len = blk_rq_payload_bytes(rq);
2783	io_dir = ((rq_data_dir(rq) == WRITE) ?
2784	NVMEFC_FCP_WRITE : NVMEFC_FCP_READ);
2785	} else {
2786	data_len = 0;
2787	io_dir = NVMEFC_FCP_NODATA;
2788	}
2789
2790
2791	return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir);
2792	}
2793
2794	static void
2795	nvme_fc_submit_async_event(struct nvme_ctrl *arg)
2796	{
2797	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: arg);
2798	struct nvme_fc_fcp_op *aen_op;
2799	blk_status_t ret;
2800
2801	if (test_bit(FCCTRL_TERMIO, &ctrl->flags))
2802	return;
2803
2804	aen_op = &ctrl->aen_ops[0];
2805
2806	ret = nvme_fc_start_fcp_op(ctrl, queue: aen_op->queue, op: aen_op, data_len: 0,
2807	io_dir: NVMEFC_FCP_NODATA);
2808	if (ret)
2809	dev_err(ctrl->ctrl.device,
2810	"failed async event work\n");
2811	}
2812
2813	static void
2814	nvme_fc_complete_rq(struct request *rq)
2815	{
2816	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2817	struct nvme_fc_ctrl *ctrl = op->ctrl;
2818
2819	atomic_set(v: &op->state, i: FCPOP_STATE_IDLE);
2820	op->flags &= ~FCOP_FLAGS_TERMIO;
2821
2822	nvme_fc_unmap_data(ctrl, rq, op);
2823	nvme_complete_rq(req: rq);
2824	nvme_fc_ctrl_put(ctrl);
2825	}
2826
2827	static void nvme_fc_map_queues(struct blk_mq_tag_set *set)
2828	{
2829	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: set->driver_data);
2830	int i;
2831
2832	for (i = 0; i < set->nr_maps; i++) {
2833	struct blk_mq_queue_map *map = &set->map[i];
2834
2835	if (!map->nr_queues) {
2836	WARN_ON(i == HCTX_TYPE_DEFAULT);
2837	continue;
2838	}
2839
2840	/* Call LLDD map queue functionality if defined */
2841	if (ctrl->lport->ops->map_queues)
2842	ctrl->lport->ops->map_queues(&ctrl->lport->localport,
2843	map);
2844	else
2845	blk_mq_map_queues(qmap: map);
2846	}
2847	}
2848
2849	static const struct blk_mq_ops nvme_fc_mq_ops = {
2850	.queue_rq = nvme_fc_queue_rq,
2851	.complete = nvme_fc_complete_rq,
2852	.init_request = nvme_fc_init_request,
2853	.exit_request = nvme_fc_exit_request,
2854	.init_hctx = nvme_fc_init_hctx,
2855	.timeout = nvme_fc_timeout,
2856	.map_queues = nvme_fc_map_queues,
2857	};
2858
2859	static int
2860	nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)
2861	{
2862	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2863	unsigned int nr_io_queues;
2864	int ret;
2865
2866	nr_io_queues = min3(opts->nr_io_queues, num_online_cpus(),
2867	ctrl->lport->ops->max_hw_queues);
2868	ret = nvme_set_queue_count(ctrl: &ctrl->ctrl, count: &nr_io_queues);
2869	if (ret) {
2870	dev_info(ctrl->ctrl.device,
2871	"set_queue_count failed: %d\n", ret);
2872	return ret;
2873	}
2874
2875	ctrl->ctrl.queue_count = nr_io_queues + 1;
2876	if (!nr_io_queues)
2877	return 0;
2878
2879	nvme_fc_init_io_queues(ctrl);
2880
2881	ret = nvme_alloc_io_tag_set(ctrl: &ctrl->ctrl, set: &ctrl->tag_set,
2882	ops: &nvme_fc_mq_ops, nr_maps: 1,
2883	struct_size_t(struct nvme_fcp_op_w_sgl, priv,
2884	ctrl->lport->ops->fcprqst_priv_sz));
2885	if (ret)
2886	return ret;
2887
2888	ret = nvme_fc_create_hw_io_queues(ctrl, qsize: ctrl->ctrl.sqsize + 1);
2889	if (ret)
2890	goto out_cleanup_tagset;
2891
2892	ret = nvme_fc_connect_io_queues(ctrl, qsize: ctrl->ctrl.sqsize + 1);
2893	if (ret)
2894	goto out_delete_hw_queues;
2895
2896	ctrl->ioq_live = true;
2897
2898	return 0;
2899
2900	out_delete_hw_queues:
2901	nvme_fc_delete_hw_io_queues(ctrl);
2902	out_cleanup_tagset:
2903	nvme_remove_io_tag_set(ctrl: &ctrl->ctrl);
2904	nvme_fc_free_io_queues(ctrl);
2905
2906	/* force put free routine to ignore io queues */
2907	ctrl->ctrl.tagset = NULL;
2908
2909	return ret;
2910	}
2911
2912	static int
2913	nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)
2914	{
2915	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2916	u32 prior_ioq_cnt = ctrl->ctrl.queue_count - 1;
2917	unsigned int nr_io_queues;
2918	int ret;
2919
2920	nr_io_queues = min3(opts->nr_io_queues, num_online_cpus(),
2921	ctrl->lport->ops->max_hw_queues);
2922	ret = nvme_set_queue_count(ctrl: &ctrl->ctrl, count: &nr_io_queues);
2923	if (ret) {
2924	dev_info(ctrl->ctrl.device,
2925	"set_queue_count failed: %d\n", ret);
2926	return ret;
2927	}
2928
2929	if (!nr_io_queues && prior_ioq_cnt) {
2930	dev_info(ctrl->ctrl.device,
2931	"Fail Reconnect: At least 1 io queue "
2932	"required (was %d)\n", prior_ioq_cnt);
2933	return -ENOSPC;
2934	}
2935
2936	ctrl->ctrl.queue_count = nr_io_queues + 1;
2937	/* check for io queues existing */
2938	if (ctrl->ctrl.queue_count == 1)
2939	return 0;
2940
2941	if (prior_ioq_cnt != nr_io_queues) {
2942	dev_info(ctrl->ctrl.device,
2943	"reconnect: revising io queue count from %d to %d\n",
2944	prior_ioq_cnt, nr_io_queues);
2945	blk_mq_update_nr_hw_queues(set: &ctrl->tag_set, nr_hw_queues: nr_io_queues);
2946	}
2947
2948	ret = nvme_fc_create_hw_io_queues(ctrl, qsize: ctrl->ctrl.sqsize + 1);
2949	if (ret)
2950	goto out_free_io_queues;
2951
2952	ret = nvme_fc_connect_io_queues(ctrl, qsize: ctrl->ctrl.sqsize + 1);
2953	if (ret)
2954	goto out_delete_hw_queues;
2955
2956	return 0;
2957
2958	out_delete_hw_queues:
2959	nvme_fc_delete_hw_io_queues(ctrl);
2960	out_free_io_queues:
2961	nvme_fc_free_io_queues(ctrl);
2962	return ret;
2963	}
2964
2965	static void
2966	nvme_fc_rport_active_on_lport(struct nvme_fc_rport *rport)
2967	{
2968	struct nvme_fc_lport *lport = rport->lport;
2969
2970	atomic_inc(v: &lport->act_rport_cnt);
2971	}
2972
2973	static void
2974	nvme_fc_rport_inactive_on_lport(struct nvme_fc_rport *rport)
2975	{
2976	struct nvme_fc_lport *lport = rport->lport;
2977	u32 cnt;
2978
2979	cnt = atomic_dec_return(v: &lport->act_rport_cnt);
2980	if (cnt == 0 && lport->localport.port_state == FC_OBJSTATE_DELETED)
2981	lport->ops->localport_delete(&lport->localport);
2982	}
2983
2984	static int
2985	nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl *ctrl)
2986	{
2987	struct nvme_fc_rport *rport = ctrl->rport;
2988	u32 cnt;
2989
2990	if (test_and_set_bit(ASSOC_ACTIVE, addr: &ctrl->flags))
2991	return 1;
2992
2993	cnt = atomic_inc_return(v: &rport->act_ctrl_cnt);
2994	if (cnt == 1)
2995	nvme_fc_rport_active_on_lport(rport);
2996
2997	return 0;
2998	}
2999
3000	static int
3001	nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl *ctrl)
3002	{
3003	struct nvme_fc_rport *rport = ctrl->rport;
3004	struct nvme_fc_lport *lport = rport->lport;
3005	u32 cnt;
3006
3007	/* clearing of ctrl->flags ASSOC_ACTIVE bit is in association delete */
3008
3009	cnt = atomic_dec_return(v: &rport->act_ctrl_cnt);
3010	if (cnt == 0) {
3011	if (rport->remoteport.port_state == FC_OBJSTATE_DELETED)
3012	lport->ops->remoteport_delete(&rport->remoteport);
3013	nvme_fc_rport_inactive_on_lport(rport);
3014	}
3015
3016	return 0;
3017	}
3018
3019	/*
3020	* This routine restarts the controller on the host side, and
3021	* on the link side, recreates the controller association.
3022	*/
3023	static int
3024	nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)
3025	{
3026	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
3027	struct nvmefc_ls_rcv_op *disls = NULL;
3028	unsigned long flags;
3029	int ret;
3030
3031	++ctrl->ctrl.nr_reconnects;
3032
3033	spin_lock_irqsave(&ctrl->rport->lock, flags);
3034	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE) {
3035	spin_unlock_irqrestore(lock: &ctrl->rport->lock, flags);
3036	return -ENODEV;
3037	}
3038
3039	if (nvme_fc_ctlr_active_on_rport(ctrl)) {
3040	spin_unlock_irqrestore(lock: &ctrl->rport->lock, flags);
3041	return -ENOTUNIQ;
3042	}
3043	spin_unlock_irqrestore(lock: &ctrl->rport->lock, flags);
3044
3045	dev_info(ctrl->ctrl.device,
3046	"NVME-FC{%d}: create association : host wwpn 0x%016llx "
3047	" rport wwpn 0x%016llx: NQN \"%s\"\n",
3048	ctrl->cnum, ctrl->lport->localport.port_name,
3049	ctrl->rport->remoteport.port_name, ctrl->ctrl.opts->subsysnqn);
3050
3051	clear_bit(ASSOC_FAILED, addr: &ctrl->flags);
3052
3053	/*
3054	* Create the admin queue
3055	*/
3056
3057	ret = __nvme_fc_create_hw_queue(ctrl, queue: &ctrl->queues[0], qidx: 0,
3058	NVME_AQ_DEPTH);
3059	if (ret)
3060	goto out_free_queue;
3061
3062	ret = nvme_fc_connect_admin_queue(ctrl, queue: &ctrl->queues[0],
3063	NVME_AQ_DEPTH, ersp_ratio: (NVME_AQ_DEPTH / 4));
3064	if (ret)
3065	goto out_delete_hw_queue;
3066
3067	ret = nvmf_connect_admin_queue(ctrl: &ctrl->ctrl);
3068	if (ret)
3069	goto out_disconnect_admin_queue;
3070
3071	set_bit(nr: NVME_FC_Q_LIVE, addr: &ctrl->queues[0].flags);
3072
3073	/*
3074	* Check controller capabilities
3075	*
3076	* todo:- add code to check if ctrl attributes changed from
3077	* prior connection values
3078	*/
3079
3080	ret = nvme_enable_ctrl(ctrl: &ctrl->ctrl);
3081	if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags))
3082	ret = -EIO;
3083	if (ret)
3084	goto out_disconnect_admin_queue;
3085
3086	ctrl->ctrl.max_segments = ctrl->lport->ops->max_sgl_segments;
3087	ctrl->ctrl.max_hw_sectors = ctrl->ctrl.max_segments <<
3088	(ilog2(SZ_4K) - 9);
3089
3090	nvme_unquiesce_admin_queue(ctrl: &ctrl->ctrl);
3091
3092	ret = nvme_init_ctrl_finish(ctrl: &ctrl->ctrl, was_suspended: false);
3093	if (ret)
3094	goto out_disconnect_admin_queue;
3095	if (test_bit(ASSOC_FAILED, &ctrl->flags)) {
3096	ret = -EIO;
3097	goto out_stop_keep_alive;
3098	}
3099	/* sanity checks */
3100
3101	/* FC-NVME does not have other data in the capsule */
3102	if (ctrl->ctrl.icdoff) {
3103	dev_err(ctrl->ctrl.device, "icdoff %d is not supported!\n",
3104	ctrl->ctrl.icdoff);
3105	ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
3106	goto out_stop_keep_alive;
3107	}
3108
3109	/* FC-NVME supports normal SGL Data Block Descriptors */
3110	if (!nvme_ctrl_sgl_supported(ctrl: &ctrl->ctrl)) {
3111	dev_err(ctrl->ctrl.device,
3112	"Mandatory sgls are not supported!\n");
3113	ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
3114	goto out_stop_keep_alive;
3115	}
3116
3117	if (opts->queue_size > ctrl->ctrl.maxcmd) {
3118	/* warn if maxcmd is lower than queue_size */
3119	dev_warn(ctrl->ctrl.device,
3120	"queue_size %zu > ctrl maxcmd %u, reducing "
3121	"to maxcmd\n",
3122	opts->queue_size, ctrl->ctrl.maxcmd);
3123	opts->queue_size = ctrl->ctrl.maxcmd;
3124	ctrl->ctrl.sqsize = opts->queue_size - 1;
3125	}
3126
3127	ret = nvme_fc_init_aen_ops(ctrl);
3128	if (ret)
3129	goto out_term_aen_ops;
3130
3131	/*
3132	* Create the io queues
3133	*/
3134
3135	if (ctrl->ctrl.queue_count > 1) {
3136	if (!ctrl->ioq_live)
3137	ret = nvme_fc_create_io_queues(ctrl);
3138	else
3139	ret = nvme_fc_recreate_io_queues(ctrl);
3140	}
3141	if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags))
3142	ret = -EIO;
3143	if (ret)
3144	goto out_term_aen_ops;
3145
3146	if (!nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_LIVE)) {
3147	ret = -EIO;
3148	goto out_term_aen_ops;
3149	}
3150
3151	ctrl->ctrl.nr_reconnects = 0;
3152	nvme_start_ctrl(ctrl: &ctrl->ctrl);
3153
3154	return 0; /* Success */
3155
3156	out_term_aen_ops:
3157	nvme_fc_term_aen_ops(ctrl);
3158	out_stop_keep_alive:
3159	nvme_stop_keep_alive(ctrl: &ctrl->ctrl);
3160	out_disconnect_admin_queue:
3161	dev_warn(ctrl->ctrl.device,
3162	"NVME-FC{%d}: create_assoc failed, assoc_id %llx ret %d\n",
3163	ctrl->cnum, ctrl->association_id, ret);
3164	/* send a Disconnect(association) LS to fc-nvme target */
3165	nvme_fc_xmt_disconnect_assoc(ctrl);
3166	spin_lock_irqsave(&ctrl->lock, flags);
3167	ctrl->association_id = 0;
3168	disls = ctrl->rcv_disconn;
3169	ctrl->rcv_disconn = NULL;
3170	spin_unlock_irqrestore(lock: &ctrl->lock, flags);
3171	if (disls)
3172	nvme_fc_xmt_ls_rsp(lsop: disls);
3173	out_delete_hw_queue:
3174	__nvme_fc_delete_hw_queue(ctrl, queue: &ctrl->queues[0], qidx: 0);
3175	out_free_queue:
3176	nvme_fc_free_queue(queue: &ctrl->queues[0]);
3177	clear_bit(ASSOC_ACTIVE, addr: &ctrl->flags);
3178	nvme_fc_ctlr_inactive_on_rport(ctrl);
3179
3180	return ret;
3181	}
3182
3183
3184	/*
3185	* This routine stops operation of the controller on the host side.
3186	* On the host os stack side: Admin and IO queues are stopped,
3187	* outstanding ios on them terminated via FC ABTS.
3188	* On the link side: the association is terminated.
3189	*/
3190	static void
3191	nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl)
3192	{
3193	struct nvmefc_ls_rcv_op *disls = NULL;
3194	unsigned long flags;
3195
3196	if (!test_and_clear_bit(ASSOC_ACTIVE, addr: &ctrl->flags))
3197	return;
3198
3199	spin_lock_irqsave(&ctrl->lock, flags);
3200	set_bit(FCCTRL_TERMIO, addr: &ctrl->flags);
3201	ctrl->iocnt = 0;
3202	spin_unlock_irqrestore(lock: &ctrl->lock, flags);
3203
3204	__nvme_fc_abort_outstanding_ios(ctrl, start_queues: false);
3205
3206	/* kill the aens as they are a separate path */
3207	nvme_fc_abort_aen_ops(ctrl);
3208
3209	/* wait for all io that had to be aborted */
3210	spin_lock_irq(lock: &ctrl->lock);
3211	wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock);
3212	clear_bit(FCCTRL_TERMIO, addr: &ctrl->flags);
3213	spin_unlock_irq(lock: &ctrl->lock);
3214
3215	nvme_fc_term_aen_ops(ctrl);
3216
3217	/*
3218	* send a Disconnect(association) LS to fc-nvme target
3219	* Note: could have been sent at top of process, but
3220	* cleaner on link traffic if after the aborts complete.
3221	* Note: if association doesn't exist, association_id will be 0
3222	*/
3223	if (ctrl->association_id)
3224	nvme_fc_xmt_disconnect_assoc(ctrl);
3225
3226	spin_lock_irqsave(&ctrl->lock, flags);
3227	ctrl->association_id = 0;
3228	disls = ctrl->rcv_disconn;
3229	ctrl->rcv_disconn = NULL;
3230	spin_unlock_irqrestore(lock: &ctrl->lock, flags);
3231	if (disls)
3232	/*
3233	* if a Disconnect Request was waiting for a response, send
3234	* now that all ABTS's have been issued (and are complete).
3235	*/
3236	nvme_fc_xmt_ls_rsp(lsop: disls);
3237
3238	if (ctrl->ctrl.tagset) {
3239	nvme_fc_delete_hw_io_queues(ctrl);
3240	nvme_fc_free_io_queues(ctrl);
3241	}
3242
3243	__nvme_fc_delete_hw_queue(ctrl, queue: &ctrl->queues[0], qidx: 0);
3244	nvme_fc_free_queue(queue: &ctrl->queues[0]);
3245
3246	/* re-enable the admin_q so anything new can fast fail */
3247	nvme_unquiesce_admin_queue(ctrl: &ctrl->ctrl);
3248
3249	/* resume the io queues so that things will fast fail */
3250	nvme_unquiesce_io_queues(ctrl: &ctrl->ctrl);
3251
3252	nvme_fc_ctlr_inactive_on_rport(ctrl);
3253	}
3254
3255	static void
3256	nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl)
3257	{
3258	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(ctrl: nctrl);
3259
3260	cancel_delayed_work_sync(dwork: &ctrl->connect_work);
3261
3262	/*
3263	* kill the association on the link side. this will block
3264	* waiting for io to terminate
3265	*/
3266	nvme_fc_delete_association(ctrl);
3267	cancel_work_sync(work: &ctrl->ioerr_work);
3268
3269	if (ctrl->ctrl.tagset)
3270	nvme_remove_io_tag_set(ctrl: &ctrl->ctrl);
3271
3272	nvme_unquiesce_admin_queue(ctrl: &ctrl->ctrl);
3273	nvme_remove_admin_tag_set(ctrl: &ctrl->ctrl);
3274	}
3275
3276	static void
3277	nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status)
3278	{
3279	struct nvme_fc_rport *rport = ctrl->rport;
3280	struct nvme_fc_remote_port *portptr = &rport->remoteport;
3281	unsigned long recon_delay = ctrl->ctrl.opts->reconnect_delay * HZ;
3282	bool recon = true;
3283
3284	if (nvme_ctrl_state(ctrl: &ctrl->ctrl) != NVME_CTRL_CONNECTING)
3285	return;
3286
3287	if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3288	dev_info(ctrl->ctrl.device,
3289	"NVME-FC{%d}: reset: Reconnect attempt failed (%d)\n",
3290	ctrl->cnum, status);
3291	} else if (time_after_eq(jiffies, rport->dev_loss_end))
3292	recon = false;
3293
3294	if (recon && nvmf_should_reconnect(ctrl: &ctrl->ctrl, status)) {
3295	if (portptr->port_state == FC_OBJSTATE_ONLINE)
3296	dev_info(ctrl->ctrl.device,
3297	"NVME-FC{%d}: Reconnect attempt in %ld "
3298	"seconds\n",
3299	ctrl->cnum, recon_delay / HZ);
3300	else if (time_after(jiffies + recon_delay, rport->dev_loss_end))
3301	recon_delay = rport->dev_loss_end - jiffies;
3302
3303	queue_delayed_work(wq: nvme_wq, dwork: &ctrl->connect_work, delay: recon_delay);
3304	} else {
3305	if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3306	if (status > 0 && (status & NVME_STATUS_DNR))
3307	dev_warn(ctrl->ctrl.device,
3308	"NVME-FC{%d}: reconnect failure\n",
3309	ctrl->cnum);
3310	else
3311	dev_warn(ctrl->ctrl.device,
3312	"NVME-FC{%d}: Max reconnect attempts "
3313	"(%d) reached.\n",
3314	ctrl->cnum, ctrl->ctrl.nr_reconnects);
3315	} else
3316	dev_warn(ctrl->ctrl.device,
3317	"NVME-FC{%d}: dev_loss_tmo (%d) expired "
3318	"while waiting for remoteport connectivity.\n",
3319	ctrl->cnum, min_t(int, portptr->dev_loss_tmo,
3320	(ctrl->ctrl.opts->max_reconnects *
3321	ctrl->ctrl.opts->reconnect_delay)));
3322	WARN_ON(nvme_delete_ctrl(&ctrl->ctrl));
3323	}
3324	}
3325
3326	static void
3327	nvme_fc_reset_ctrl_work(struct work_struct *work)
3328	{
3329	struct nvme_fc_ctrl *ctrl =
3330	container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);
3331
3332	nvme_stop_ctrl(ctrl: &ctrl->ctrl);
3333
3334	/* will block will waiting for io to terminate */
3335	nvme_fc_delete_association(ctrl);
3336
3337	if (!nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_CONNECTING))
3338	dev_err(ctrl->ctrl.device,
3339	"NVME-FC{%d}: error_recovery: Couldn't change state "
3340	"to CONNECTING\n", ctrl->cnum);
3341
3342	if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
3343	if (!queue_delayed_work(wq: nvme_wq, dwork: &ctrl->connect_work, delay: 0)) {
3344	dev_err(ctrl->ctrl.device,
3345	"NVME-FC{%d}: failed to schedule connect "
3346	"after reset\n", ctrl->cnum);
3347	} else {
3348	flush_delayed_work(dwork: &ctrl->connect_work);
3349	}
3350	} else {
3351	nvme_fc_reconnect_or_delete(ctrl, status: -ENOTCONN);
3352	}
3353	}
3354
3355
3356	static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = {
3357	.name = "fc",
3358	.module = THIS_MODULE,
3359	.flags = NVME_F_FABRICS,
3360	.reg_read32 = nvmf_reg_read32,
3361	.reg_read64 = nvmf_reg_read64,
3362	.reg_write32 = nvmf_reg_write32,
3363	.subsystem_reset = nvmf_subsystem_reset,
3364	.free_ctrl = nvme_fc_free_ctrl,
3365	.submit_async_event = nvme_fc_submit_async_event,
3366	.delete_ctrl = nvme_fc_delete_ctrl,
3367	.get_address = nvmf_get_address,
3368	.get_virt_boundary = nvmf_get_virt_boundary,
3369	};
3370
3371	static void
3372	nvme_fc_connect_ctrl_work(struct work_struct *work)
3373	{
3374	int ret;
3375
3376	struct nvme_fc_ctrl *ctrl =
3377	container_of(to_delayed_work(work),
3378	struct nvme_fc_ctrl, connect_work);
3379
3380	ret = nvme_fc_create_association(ctrl);
3381	if (ret)
3382	nvme_fc_reconnect_or_delete(ctrl, status: ret);
3383	else
3384	dev_info(ctrl->ctrl.device,
3385	"NVME-FC{%d}: controller connect complete\n",
3386	ctrl->cnum);
3387	}
3388
3389
3390	static const struct blk_mq_ops nvme_fc_admin_mq_ops = {
3391	.queue_rq = nvme_fc_queue_rq,
3392	.complete = nvme_fc_complete_rq,
3393	.init_request = nvme_fc_init_request,
3394	.exit_request = nvme_fc_exit_request,
3395	.init_hctx = nvme_fc_init_admin_hctx,
3396	.timeout = nvme_fc_timeout,
3397	};
3398
3399
3400	/*
3401	* Fails a controller request if it matches an existing controller
3402	* (association) with the same tuple:
3403	* <Host NQN, Host ID, local FC port, remote FC port, SUBSYS NQN>
3404	*
3405	* The ports don't need to be compared as they are intrinsically
3406	* already matched by the port pointers supplied.
3407	*/
3408	static bool
3409	nvme_fc_existing_controller(struct nvme_fc_rport *rport,
3410	struct nvmf_ctrl_options *opts)
3411	{
3412	struct nvme_fc_ctrl *ctrl;
3413	unsigned long flags;
3414	bool found = false;
3415
3416	spin_lock_irqsave(&rport->lock, flags);
3417	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3418	found = nvmf_ctlr_matches_baseopts(ctrl: &ctrl->ctrl, opts);
3419	if (found)
3420	break;
3421	}
3422	spin_unlock_irqrestore(lock: &rport->lock, flags);
3423
3424	return found;
3425	}
3426
3427	static struct nvme_fc_ctrl *
3428	nvme_fc_alloc_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
3429	struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
3430	{
3431	struct nvme_fc_ctrl *ctrl;
3432	int ret, idx, ctrl_loss_tmo;
3433
3434	if (!(rport->remoteport.port_role &
3435	(FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET))) {
3436	ret = -EBADR;
3437	goto out_fail;
3438	}
3439
3440	if (!opts->duplicate_connect &&
3441	nvme_fc_existing_controller(rport, opts)) {
3442	ret = -EALREADY;
3443	goto out_fail;
3444	}
3445
3446	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
3447	if (!ctrl) {
3448	ret = -ENOMEM;
3449	goto out_fail;
3450	}
3451
3452	idx = ida_alloc(ida: &nvme_fc_ctrl_cnt, GFP_KERNEL);
3453	if (idx < 0) {
3454	ret = -ENOSPC;
3455	goto out_free_ctrl;
3456	}
3457
3458	/*
3459	* if ctrl_loss_tmo is being enforced and the default reconnect delay
3460	* is being used, change to a shorter reconnect delay for FC.
3461	*/
3462	if (opts->max_reconnects != -1 &&
3463	opts->reconnect_delay == NVMF_DEF_RECONNECT_DELAY &&
3464	opts->reconnect_delay > NVME_FC_DEFAULT_RECONNECT_TMO) {
3465	ctrl_loss_tmo = opts->max_reconnects * opts->reconnect_delay;
3466	opts->reconnect_delay = NVME_FC_DEFAULT_RECONNECT_TMO;
3467	opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3468	opts->reconnect_delay);
3469	}
3470
3471	ctrl->ctrl.opts = opts;
3472	ctrl->ctrl.nr_reconnects = 0;
3473	INIT_LIST_HEAD(list: &ctrl->ctrl_list);
3474	ctrl->lport = lport;
3475	ctrl->rport = rport;
3476	ctrl->dev = lport->dev;
3477	ctrl->cnum = idx;
3478	ctrl->ioq_live = false;
3479	init_waitqueue_head(&ctrl->ioabort_wait);
3480
3481	get_device(dev: ctrl->dev);
3482	kref_init(kref: &ctrl->ref);
3483
3484	INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
3485	INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
3486	INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work);
3487	spin_lock_init(&ctrl->lock);
3488
3489	/* io queue count */
3490	ctrl->ctrl.queue_count = min_t(unsigned int,
3491	opts->nr_io_queues,
3492	lport->ops->max_hw_queues);
3493	ctrl->ctrl.queue_count++; /* +1 for admin queue */
3494
3495	ctrl->ctrl.sqsize = opts->queue_size - 1;
3496	ctrl->ctrl.kato = opts->kato;
3497	ctrl->ctrl.cntlid = 0xffff;
3498
3499	ret = -ENOMEM;
3500	ctrl->queues = kcalloc(ctrl->ctrl.queue_count,
3501	sizeof(struct nvme_fc_queue), GFP_KERNEL);
3502	if (!ctrl->queues)
3503	goto out_free_ida;
3504
3505	nvme_fc_init_queue(ctrl, idx: 0);
3506
3507	/*
3508	* Would have been nice to init io queues tag set as well.
3509	* However, we require interaction from the controller
3510	* for max io queue count before we can do so.
3511	* Defer this to the connect path.
3512	*/
3513
3514	ret = nvme_init_ctrl(ctrl: &ctrl->ctrl, dev, ops: &nvme_fc_ctrl_ops, quirks: 0);
3515	if (ret)
3516	goto out_free_queues;
3517	if (lport->dev)
3518	ctrl->ctrl.numa_node = dev_to_node(dev: lport->dev);
3519
3520	return ctrl;
3521
3522	out_free_queues:
3523	kfree(objp: ctrl->queues);
3524	out_free_ida:
3525	put_device(dev: ctrl->dev);
3526	ida_free(&nvme_fc_ctrl_cnt, id: ctrl->cnum);
3527	out_free_ctrl:
3528	kfree(objp: ctrl);
3529	out_fail:
3530	/* exit via here doesn't follow ctlr ref points */
3531	return ERR_PTR(error: ret);
3532	}
3533
3534	static struct nvme_ctrl *
3535	nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
3536	struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
3537	{
3538	struct nvme_fc_ctrl *ctrl;
3539	unsigned long flags;
3540	int ret;
3541
3542	ctrl = nvme_fc_alloc_ctrl(dev, opts, lport, rport);
3543	if (IS_ERR(ptr: ctrl))
3544	return ERR_CAST(ptr: ctrl);
3545
3546	ret = nvme_add_ctrl(ctrl: &ctrl->ctrl);
3547	if (ret)
3548	goto out_put_ctrl;
3549
3550	ret = nvme_alloc_admin_tag_set(ctrl: &ctrl->ctrl, set: &ctrl->admin_tag_set,
3551	ops: &nvme_fc_admin_mq_ops,
3552	struct_size_t(struct nvme_fcp_op_w_sgl, priv,
3553	ctrl->lport->ops->fcprqst_priv_sz));
3554	if (ret)
3555	goto fail_ctrl;
3556
3557	spin_lock_irqsave(&rport->lock, flags);
3558	list_add_tail(new: &ctrl->ctrl_list, head: &rport->ctrl_list);
3559	spin_unlock_irqrestore(lock: &rport->lock, flags);
3560
3561	if (!nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_CONNECTING)) {
3562	dev_err(ctrl->ctrl.device,
3563	"NVME-FC{%d}: failed to init ctrl state\n", ctrl->cnum);
3564	goto fail_ctrl;
3565	}
3566
3567	if (!queue_delayed_work(wq: nvme_wq, dwork: &ctrl->connect_work, delay: 0)) {
3568	dev_err(ctrl->ctrl.device,
3569	"NVME-FC{%d}: failed to schedule initial connect\n",
3570	ctrl->cnum);
3571	goto fail_ctrl;
3572	}
3573
3574	flush_delayed_work(dwork: &ctrl->connect_work);
3575
3576	dev_info(ctrl->ctrl.device,
3577	"NVME-FC{%d}: new ctrl: NQN \"%s\", hostnqn: %s\n",
3578	ctrl->cnum, nvmf_ctrl_subsysnqn(&ctrl->ctrl), opts->host->nqn);
3579
3580	return &ctrl->ctrl;
3581
3582	fail_ctrl:
3583	nvme_change_ctrl_state(ctrl: &ctrl->ctrl, new_state: NVME_CTRL_DELETING);
3584	cancel_work_sync(work: &ctrl->ioerr_work);
3585	cancel_work_sync(work: &ctrl->ctrl.reset_work);
3586	cancel_delayed_work_sync(dwork: &ctrl->connect_work);
3587
3588	ctrl->ctrl.opts = NULL;
3589
3590	if (ctrl->ctrl.admin_tagset)
3591	nvme_remove_admin_tag_set(ctrl: &ctrl->ctrl);
3592	/* initiate nvme ctrl ref counting teardown */
3593	nvme_uninit_ctrl(ctrl: &ctrl->ctrl);
3594
3595	out_put_ctrl:
3596	/* Remove core ctrl ref. */
3597	nvme_put_ctrl(ctrl: &ctrl->ctrl);
3598
3599	/* as we're past the point where we transition to the ref
3600	* counting teardown path, if we return a bad pointer here,
3601	* the calling routine, thinking it's prior to the
3602	* transition, will do an rport put. Since the teardown
3603	* path also does a rport put, we do an extra get here to
3604	* so proper order/teardown happens.
3605	*/
3606	nvme_fc_rport_get(rport);
3607
3608	return ERR_PTR(error: -EIO);
3609	}
3610
3611	struct nvmet_fc_traddr {
3612	u64 nn;
3613	u64 pn;
3614	};
3615
3616	static int
3617	__nvme_fc_parse_u64(substring_t *sstr, u64 *val)
3618	{
3619	u64 token64;
3620
3621	if (match_u64(sstr, result: &token64))
3622	return -EINVAL;
3623	*val = token64;
3624
3625	return 0;
3626	}
3627
3628	/*
3629	* This routine validates and extracts the WWN's from the TRADDR string.
3630	* As kernel parsers need the 0x to determine number base, universally
3631	* build string to parse with 0x prefix before parsing name strings.
3632	*/
3633	static int
3634	nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
3635	{
3636	char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
3637	substring_t wwn = { name, &name[sizeof(name)-1] };
3638	int nnoffset, pnoffset;
3639
3640	/* validate if string is one of the 2 allowed formats */
3641	if (strnlen(p: buf, maxlen: blen) == NVME_FC_TRADDR_MAXLENGTH &&
3642	!strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
3643	!strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
3644	"pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
3645	nnoffset = NVME_FC_TRADDR_OXNNLEN;
3646	pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
3647	NVME_FC_TRADDR_OXNNLEN;
3648	} else if ((strnlen(p: buf, maxlen: blen) == NVME_FC_TRADDR_MINLENGTH &&
3649	!strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
3650	!strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
3651	"pn-", NVME_FC_TRADDR_NNLEN))) {
3652	nnoffset = NVME_FC_TRADDR_NNLEN;
3653	pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
3654	} else
3655	goto out_einval;
3656
3657	name[0] = '0';
3658	name[1] = 'x';
3659	name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
3660
3661	memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3662	if (__nvme_fc_parse_u64(sstr: &wwn, val: &traddr->nn))
3663	goto out_einval;
3664
3665	memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3666	if (__nvme_fc_parse_u64(sstr: &wwn, val: &traddr->pn))
3667	goto out_einval;
3668
3669	return 0;
3670
3671	out_einval:
3672	pr_warn("%s: bad traddr string\n", __func__);
3673	return -EINVAL;
3674	}
3675
3676	static struct nvme_ctrl *
3677	nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts)
3678	{
3679	struct nvme_fc_lport *lport;
3680	struct nvme_fc_rport *rport;
3681	struct nvme_ctrl *ctrl;
3682	struct nvmet_fc_traddr laddr = { 0L, 0L };
3683	struct nvmet_fc_traddr raddr = { 0L, 0L };
3684	unsigned long flags;
3685	int ret;
3686
3687	ret = nvme_fc_parse_traddr(traddr: &raddr, buf: opts->traddr, NVMF_TRADDR_SIZE);
3688	if (ret || !raddr.nn || !raddr.pn)
3689	return ERR_PTR(error: -EINVAL);
3690
3691	ret = nvme_fc_parse_traddr(traddr: &laddr, buf: opts->host_traddr, NVMF_TRADDR_SIZE);
3692	if (ret || !laddr.nn || !laddr.pn)
3693	return ERR_PTR(error: -EINVAL);
3694
3695	/* find the host and remote ports to connect together */
3696	spin_lock_irqsave(&nvme_fc_lock, flags);
3697	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3698	if (lport->localport.node_name != laddr.nn ||
3699	lport->localport.port_name != laddr.pn ||
3700	lport->localport.port_state != FC_OBJSTATE_ONLINE)
3701	continue;
3702
3703	list_for_each_entry(rport, &lport->endp_list, endp_list) {
3704	if (rport->remoteport.node_name != raddr.nn ||
3705	rport->remoteport.port_name != raddr.pn ||
3706	rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3707	continue;
3708
3709	/* if fail to get reference fall through. Will error */
3710	if (!nvme_fc_rport_get(rport))
3711	break;
3712
3713	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
3714
3715	ctrl = nvme_fc_init_ctrl(dev, opts, lport, rport);
3716	if (IS_ERR(ptr: ctrl))
3717	nvme_fc_rport_put(rport);
3718	return ctrl;
3719	}
3720	}
3721	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
3722
3723	pr_warn("%s: %s - %s combination not found\n",
3724	__func__, opts->traddr, opts->host_traddr);
3725	return ERR_PTR(error: -ENOENT);
3726	}
3727
3728
3729	static struct nvmf_transport_ops nvme_fc_transport = {
3730	.name = "fc",
3731	.module = THIS_MODULE,
3732	.required_opts = NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR,
3733	.allowed_opts = NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_CTRL_LOSS_TMO,
3734	.create_ctrl = nvme_fc_create_ctrl,
3735	};
3736
3737	/* Arbitrary successive failures max. With lots of subsystems could be high */
3738	#define DISCOVERY_MAX_FAIL 20
3739
3740	static ssize_t nvme_fc_nvme_discovery_store(struct device *dev,
3741	struct device_attribute *attr, const char *buf, size_t count)
3742	{
3743	unsigned long flags;
3744	LIST_HEAD(local_disc_list);
3745	struct nvme_fc_lport *lport;
3746	struct nvme_fc_rport *rport;
3747	int failcnt = 0;
3748
3749	spin_lock_irqsave(&nvme_fc_lock, flags);
3750	restart:
3751	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3752	list_for_each_entry(rport, &lport->endp_list, endp_list) {
3753	if (!nvme_fc_lport_get(lport))
3754	continue;
3755	if (!nvme_fc_rport_get(rport)) {
3756	/*
3757	* This is a temporary condition. Upon restart
3758	* this rport will be gone from the list.
3759	*
3760	* Revert the lport put and retry. Anything
3761	* added to the list already will be skipped (as
3762	* they are no longer list_empty). Loops should
3763	* resume at rports that were not yet seen.
3764	*/
3765	nvme_fc_lport_put(lport);
3766
3767	if (failcnt++ < DISCOVERY_MAX_FAIL)
3768	goto restart;
3769
3770	pr_err("nvme_discovery: too many reference "
3771	"failures\n");
3772	goto process_local_list;
3773	}
3774	if (list_empty(head: &rport->disc_list))
3775	list_add_tail(new: &rport->disc_list,
3776	head: &local_disc_list);
3777	}
3778	}
3779
3780	process_local_list:
3781	while (!list_empty(head: &local_disc_list)) {
3782	rport = list_first_entry(&local_disc_list,
3783	struct nvme_fc_rport, disc_list);
3784	list_del_init(entry: &rport->disc_list);
3785	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
3786
3787	lport = rport->lport;
3788	/* signal discovery. Won't hurt if it repeats */
3789	nvme_fc_signal_discovery_scan(lport, rport);
3790	nvme_fc_rport_put(rport);
3791	nvme_fc_lport_put(lport);
3792
3793	spin_lock_irqsave(&nvme_fc_lock, flags);
3794	}
3795	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
3796
3797	return count;
3798	}
3799
3800	static DEVICE_ATTR(nvme_discovery, 0200, NULL, nvme_fc_nvme_discovery_store);
3801
3802	#ifdef CONFIG_BLK_CGROUP_FC_APPID
3803	/* Parse the cgroup id from a buf and return the length of cgrpid */
3804	static int fc_parse_cgrpid(const char *buf, u64 *id)
3805	{
3806	char cgrp_id[16+1];
3807	int cgrpid_len, j;
3808
3809	memset(cgrp_id, 0x0, sizeof(cgrp_id));
3810	for (cgrpid_len = 0, j = 0; cgrpid_len < 17; cgrpid_len++) {
3811	if (buf[cgrpid_len] != ':')
3812	cgrp_id[cgrpid_len] = buf[cgrpid_len];
3813	else {
3814	j = 1;
3815	break;
3816	}
3817	}
3818	if (!j)
3819	return -EINVAL;
3820	if (kstrtou64(s: cgrp_id, base: 16, res: id) < 0)
3821	return -EINVAL;
3822	return cgrpid_len;
3823	}
3824
3825	/*
3826	* Parse and update the appid in the blkcg associated with the cgroupid.
3827	*/
3828	static ssize_t fc_appid_store(struct device *dev,
3829	struct device_attribute *attr, const char *buf, size_t count)
3830	{
3831	size_t orig_count = count;
3832	u64 cgrp_id;
3833	int appid_len = 0;
3834	int cgrpid_len = 0;
3835	char app_id[FC_APPID_LEN];
3836	int ret = 0;
3837
3838	if (buf[count-1] == '\n')
3839	count--;
3840
3841	if ((count > (16+1+FC_APPID_LEN)) || (!strchr(buf, ':')))
3842	return -EINVAL;
3843
3844	cgrpid_len = fc_parse_cgrpid(buf, id: &cgrp_id);
3845	if (cgrpid_len < 0)
3846	return -EINVAL;
3847	appid_len = count - cgrpid_len - 1;
3848	if (appid_len > FC_APPID_LEN)
3849	return -EINVAL;
3850
3851	memset(app_id, 0x0, sizeof(app_id));
3852	memcpy(app_id, &buf[cgrpid_len+1], appid_len);
3853	ret = blkcg_set_fc_appid(app_id, cgrp_id, app_id_len: sizeof(app_id));
3854	if (ret < 0)
3855	return ret;
3856	return orig_count;
3857	}
3858	static DEVICE_ATTR(appid_store, 0200, NULL, fc_appid_store);
3859	#endif /* CONFIG_BLK_CGROUP_FC_APPID */
3860
3861	static struct attribute *nvme_fc_attrs[] = {
3862	&dev_attr_nvme_discovery.attr,
3863	#ifdef CONFIG_BLK_CGROUP_FC_APPID
3864	&dev_attr_appid_store.attr,
3865	#endif
3866	NULL
3867	};
3868
3869	static const struct attribute_group nvme_fc_attr_group = {
3870	.attrs = nvme_fc_attrs,
3871	};
3872
3873	static const struct attribute_group *nvme_fc_attr_groups[] = {
3874	&nvme_fc_attr_group,
3875	NULL
3876	};
3877
3878	static struct class fc_class = {
3879	.name = "fc",
3880	.dev_groups = nvme_fc_attr_groups,
3881	};
3882
3883	static int __init nvme_fc_init_module(void)
3884	{
3885	int ret;
3886
3887	/*
3888	* NOTE:
3889	* It is expected that in the future the kernel will combine
3890	* the FC-isms that are currently under scsi and now being
3891	* added to by NVME into a new standalone FC class. The SCSI
3892	* and NVME protocols and their devices would be under this
3893	* new FC class.
3894	*
3895	* As we need something to post FC-specific udev events to,
3896	* specifically for nvme probe events, start by creating the
3897	* new device class. When the new standalone FC class is
3898	* put in place, this code will move to a more generic
3899	* location for the class.
3900	*/
3901	ret = class_register(class: &fc_class);
3902	if (ret) {
3903	pr_err("couldn't register class fc\n");
3904	return ret;
3905	}
3906
3907	/*
3908	* Create a device for the FC-centric udev events
3909	*/
3910	fc_udev_device = device_create(cls: &fc_class, NULL, MKDEV(0, 0), NULL,
3911	fmt: "fc_udev_device");
3912	if (IS_ERR(ptr: fc_udev_device)) {
3913	pr_err("couldn't create fc_udev device!\n");
3914	ret = PTR_ERR(ptr: fc_udev_device);
3915	goto out_destroy_class;
3916	}
3917
3918	ret = nvmf_register_transport(ops: &nvme_fc_transport);
3919	if (ret)
3920	goto out_destroy_device;
3921
3922	return 0;
3923
3924	out_destroy_device:
3925	device_destroy(cls: &fc_class, MKDEV(0, 0));
3926	out_destroy_class:
3927	class_unregister(class: &fc_class);
3928
3929	return ret;
3930	}
3931
3932	static void
3933	nvme_fc_delete_controllers(struct nvme_fc_rport *rport)
3934	{
3935	struct nvme_fc_ctrl *ctrl;
3936
3937	spin_lock(lock: &rport->lock);
3938	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3939	dev_warn(ctrl->ctrl.device,
3940	"NVME-FC{%d}: transport unloading: deleting ctrl\n",
3941	ctrl->cnum);
3942	nvme_delete_ctrl(ctrl: &ctrl->ctrl);
3943	}
3944	spin_unlock(lock: &rport->lock);
3945	}
3946
3947	static void __exit nvme_fc_exit_module(void)
3948	{
3949	struct nvme_fc_lport *lport;
3950	struct nvme_fc_rport *rport;
3951	unsigned long flags;
3952
3953	spin_lock_irqsave(&nvme_fc_lock, flags);
3954	list_for_each_entry(lport, &nvme_fc_lport_list, port_list)
3955	list_for_each_entry(rport, &lport->endp_list, endp_list)
3956	nvme_fc_delete_controllers(rport);
3957	spin_unlock_irqrestore(lock: &nvme_fc_lock, flags);
3958	flush_workqueue(nvme_delete_wq);
3959
3960	nvmf_unregister_transport(ops: &nvme_fc_transport);
3961
3962	device_destroy(cls: &fc_class, MKDEV(0, 0));
3963	class_unregister(class: &fc_class);
3964	}
3965
3966	module_init(nvme_fc_init_module);
3967	module_exit(nvme_fc_exit_module);
3968
3969	MODULE_DESCRIPTION("NVMe host FC transport driver");
3970	MODULE_LICENSE("GPL v2");
3971