1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* NVMe over Fabrics common host code.
4	* Copyright (c) 2015-2016 HGST, a Western Digital Company.
5	*/
6	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7	#include <linux/init.h>
8	#include <linux/miscdevice.h>
9	#include <linux/module.h>
10	#include <linux/mutex.h>
11	#include <linux/parser.h>
12	#include <linux/seq_file.h>
13	#include "nvme.h"
14	#include "fabrics.h"
15	#include <linux/nvme-keyring.h>
16
17	static LIST_HEAD(nvmf_transports);
18	static DECLARE_RWSEM(nvmf_transports_rwsem);
19
20	static LIST_HEAD(nvmf_hosts);
21	static DEFINE_MUTEX(nvmf_hosts_mutex);
22
23	static struct nvmf_host *nvmf_default_host;
24
25	static struct nvmf_host *nvmf_host_alloc(const char *hostnqn, uuid_t *id)
26	{
27	struct nvmf_host *host;
28
29	host = kmalloc(sizeof(*host), GFP_KERNEL);
30	if (!host)
31	return NULL;
32
33	kref_init(kref: &host->ref);
34	uuid_copy(dst: &host->id, src: id);
35	strscpy(host->nqn, hostnqn, NVMF_NQN_SIZE);
36
37	return host;
38	}
39
40	static struct nvmf_host *nvmf_host_add(const char *hostnqn, uuid_t *id)
41	{
42	struct nvmf_host *host;
43
44	mutex_lock(&nvmf_hosts_mutex);
45
46	/*
47	* We have defined a host as how it is perceived by the target.
48	* Therefore, we don't allow different Host NQNs with the same Host ID.
49	* Similarly, we do not allow the usage of the same Host NQN with
50	* different Host IDs. This'll maintain unambiguous host identification.
51	*/
52	list_for_each_entry(host, &nvmf_hosts, list) {
53	bool same_hostnqn = !strcmp(host->nqn, hostnqn);
54	bool same_hostid = uuid_equal(u1: &host->id, u2: id);
55
56	if (same_hostnqn && same_hostid) {
57	kref_get(kref: &host->ref);
58	goto out_unlock;
59	}
60	if (same_hostnqn) {
61	pr_err("found same hostnqn %s but different hostid %pUb\n",
62	hostnqn, id);
63	host = ERR_PTR(error: -EINVAL);
64	goto out_unlock;
65	}
66	if (same_hostid) {
67	pr_err("found same hostid %pUb but different hostnqn %s\n",
68	id, hostnqn);
69	host = ERR_PTR(error: -EINVAL);
70	goto out_unlock;
71	}
72	}
73
74	host = nvmf_host_alloc(hostnqn, id);
75	if (!host) {
76	host = ERR_PTR(error: -ENOMEM);
77	goto out_unlock;
78	}
79
80	list_add_tail(new: &host->list, head: &nvmf_hosts);
81	out_unlock:
82	mutex_unlock(lock: &nvmf_hosts_mutex);
83	return host;
84	}
85
86	static struct nvmf_host *nvmf_host_default(void)
87	{
88	struct nvmf_host *host;
89	char nqn[NVMF_NQN_SIZE];
90	uuid_t id;
91
92	uuid_gen(u: &id);
93	snprintf(buf: nqn, NVMF_NQN_SIZE,
94	fmt: "nqn.2014-08.org.nvmexpress:uuid:%pUb", &id);
95
96	host = nvmf_host_alloc(hostnqn: nqn, id: &id);
97	if (!host)
98	return NULL;
99
100	mutex_lock(&nvmf_hosts_mutex);
101	list_add_tail(new: &host->list, head: &nvmf_hosts);
102	mutex_unlock(lock: &nvmf_hosts_mutex);
103
104	return host;
105	}
106
107	static void nvmf_host_destroy(struct kref *ref)
108	{
109	struct nvmf_host *host = container_of(ref, struct nvmf_host, ref);
110
111	mutex_lock(&nvmf_hosts_mutex);
112	list_del(entry: &host->list);
113	mutex_unlock(lock: &nvmf_hosts_mutex);
114
115	kfree(objp: host);
116	}
117
118	static void nvmf_host_put(struct nvmf_host *host)
119	{
120	if (host)
121	kref_put(kref: &host->ref, release: nvmf_host_destroy);
122	}
123
124	/**
125	* nvmf_get_address() - Get address/port
126	* @ctrl: Host NVMe controller instance which we got the address
127	* @buf: OUTPUT parameter that will contain the address/port
128	* @size: buffer size
129	*/
130	int nvmf_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
131	{
132	int len = 0;
133
134	if (ctrl->opts->mask & NVMF_OPT_TRADDR)
135	len += scnprintf(buf, size, fmt: "traddr=%s", ctrl->opts->traddr);
136	if (ctrl->opts->mask & NVMF_OPT_TRSVCID)
137	len += scnprintf(buf: buf + len, size: size - len, fmt: "%strsvcid=%s",
138	(len) ? "," : "", ctrl->opts->trsvcid);
139	if (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)
140	len += scnprintf(buf: buf + len, size: size - len, fmt: "%shost_traddr=%s",
141	(len) ? "," : "", ctrl->opts->host_traddr);
142	if (ctrl->opts->mask & NVMF_OPT_HOST_IFACE)
143	len += scnprintf(buf: buf + len, size: size - len, fmt: "%shost_iface=%s",
144	(len) ? "," : "", ctrl->opts->host_iface);
145	len += scnprintf(buf: buf + len, size: size - len, fmt: "\n");
146
147	return len;
148	}
149	EXPORT_SYMBOL_GPL(nvmf_get_address);
150
151	/**
152	* nvmf_reg_read32() - NVMe Fabrics "Property Get" API function.
153	* @ctrl: Host NVMe controller instance maintaining the admin
154	* queue used to submit the property read command to
155	* the allocated NVMe controller resource on the target system.
156	* @off: Starting offset value of the targeted property
157	* register (see the fabrics section of the NVMe standard).
158	* @val: OUTPUT parameter that will contain the value of
159	* the property after a successful read.
160	*
161	* Used by the host system to retrieve a 32-bit capsule property value
162	* from an NVMe controller on the target system.
163	*
164	* ("Capsule property" is an "PCIe register concept" applied to the
165	* NVMe fabrics space.)
166	*
167	* Return:
168	* 0: successful read
169	* > 0: NVMe error status code
170	* < 0: Linux errno error code
171	*/
172	int nvmf_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
173	{
174	struct nvme_command cmd = { };
175	union nvme_result res;
176	int ret;
177
178	cmd.prop_get.opcode = nvme_fabrics_command;
179	cmd.prop_get.fctype = nvme_fabrics_type_property_get;
180	cmd.prop_get.offset = cpu_to_le32(off);
181
182	ret = __nvme_submit_sync_cmd(q: ctrl->fabrics_q, cmd: &cmd, result: &res, NULL, bufflen: 0,
183	NVME_QID_ANY, flags: NVME_SUBMIT_RESERVED);
184
185	if (ret >= 0)
186	*val = le64_to_cpu(res.u64);
187	if (unlikely(ret != 0))
188	dev_err(ctrl->device,
189	"Property Get error: %d, offset %#x\n",
190	ret > 0 ? ret & ~NVME_STATUS_DNR : ret, off);
191
192	return ret;
193	}
194	EXPORT_SYMBOL_GPL(nvmf_reg_read32);
195
196	/**
197	* nvmf_reg_read64() - NVMe Fabrics "Property Get" API function.
198	* @ctrl: Host NVMe controller instance maintaining the admin
199	* queue used to submit the property read command to
200	* the allocated controller resource on the target system.
201	* @off: Starting offset value of the targeted property
202	* register (see the fabrics section of the NVMe standard).
203	* @val: OUTPUT parameter that will contain the value of
204	* the property after a successful read.
205	*
206	* Used by the host system to retrieve a 64-bit capsule property value
207	* from an NVMe controller on the target system.
208	*
209	* ("Capsule property" is an "PCIe register concept" applied to the
210	* NVMe fabrics space.)
211	*
212	* Return:
213	* 0: successful read
214	* > 0: NVMe error status code
215	* < 0: Linux errno error code
216	*/
217	int nvmf_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
218	{
219	struct nvme_command cmd = { };
220	union nvme_result res;
221	int ret;
222
223	cmd.prop_get.opcode = nvme_fabrics_command;
224	cmd.prop_get.fctype = nvme_fabrics_type_property_get;
225	cmd.prop_get.attrib = 1;
226	cmd.prop_get.offset = cpu_to_le32(off);
227
228	ret = __nvme_submit_sync_cmd(q: ctrl->fabrics_q, cmd: &cmd, result: &res, NULL, bufflen: 0,
229	NVME_QID_ANY, flags: NVME_SUBMIT_RESERVED);
230
231	if (ret >= 0)
232	*val = le64_to_cpu(res.u64);
233	if (unlikely(ret != 0))
234	dev_err(ctrl->device,
235	"Property Get error: %d, offset %#x\n",
236	ret > 0 ? ret & ~NVME_STATUS_DNR : ret, off);
237	return ret;
238	}
239	EXPORT_SYMBOL_GPL(nvmf_reg_read64);
240
241	/**
242	* nvmf_reg_write32() - NVMe Fabrics "Property Write" API function.
243	* @ctrl: Host NVMe controller instance maintaining the admin
244	* queue used to submit the property read command to
245	* the allocated NVMe controller resource on the target system.
246	* @off: Starting offset value of the targeted property
247	* register (see the fabrics section of the NVMe standard).
248	* @val: Input parameter that contains the value to be
249	* written to the property.
250	*
251	* Used by the NVMe host system to write a 32-bit capsule property value
252	* to an NVMe controller on the target system.
253	*
254	* ("Capsule property" is an "PCIe register concept" applied to the
255	* NVMe fabrics space.)
256	*
257	* Return:
258	* 0: successful write
259	* > 0: NVMe error status code
260	* < 0: Linux errno error code
261	*/
262	int nvmf_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
263	{
264	struct nvme_command cmd = { };
265	int ret;
266
267	cmd.prop_set.opcode = nvme_fabrics_command;
268	cmd.prop_set.fctype = nvme_fabrics_type_property_set;
269	cmd.prop_set.attrib = 0;
270	cmd.prop_set.offset = cpu_to_le32(off);
271	cmd.prop_set.value = cpu_to_le64(val);
272
273	ret = __nvme_submit_sync_cmd(q: ctrl->fabrics_q, cmd: &cmd, NULL, NULL, bufflen: 0,
274	NVME_QID_ANY, flags: NVME_SUBMIT_RESERVED);
275	if (unlikely(ret))
276	dev_err(ctrl->device,
277	"Property Set error: %d, offset %#x\n",
278	ret > 0 ? ret & ~NVME_STATUS_DNR : ret, off);
279	return ret;
280	}
281	EXPORT_SYMBOL_GPL(nvmf_reg_write32);
282
283	int nvmf_subsystem_reset(struct nvme_ctrl *ctrl)
284	{
285	int ret;
286
287	if (!nvme_wait_reset(ctrl))
288	return -EBUSY;
289
290	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, NVME_SUBSYS_RESET);
291	if (ret)
292	return ret;
293
294	return nvme_try_sched_reset(ctrl);
295	}
296	EXPORT_SYMBOL_GPL(nvmf_subsystem_reset);
297
298	/**
299	* nvmf_log_connect_error() - Error-parsing-diagnostic print out function for
300	* connect() errors.
301	* @ctrl: The specific /dev/nvmeX device that had the error.
302	* @errval: Error code to be decoded in a more human-friendly
303	* printout.
304	* @offset: For use with the NVMe error code
305	* NVME_SC_CONNECT_INVALID_PARAM.
306	* @cmd: This is the SQE portion of a submission capsule.
307	* @data: This is the "Data" portion of a submission capsule.
308	*/
309	static void nvmf_log_connect_error(struct nvme_ctrl *ctrl,
310	int errval, int offset, struct nvme_command *cmd,
311	struct nvmf_connect_data *data)
312	{
313	int err_sctype = errval & ~NVME_STATUS_DNR;
314
315	if (errval < 0) {
316	dev_err(ctrl->device,
317	"Connect command failed, errno: %d\n", errval);
318	return;
319	}
320
321	switch (err_sctype) {
322	case NVME_SC_CONNECT_INVALID_PARAM:
323	if (offset >> 16) {
324	char *inv_data = "Connect Invalid Data Parameter";
325
326	switch (offset & 0xffff) {
327	case (offsetof(struct nvmf_connect_data, cntlid)):
328	dev_err(ctrl->device,
329	"%s, cntlid: %d\n",
330	inv_data, data->cntlid);
331	break;
332	case (offsetof(struct nvmf_connect_data, hostnqn)):
333	dev_err(ctrl->device,
334	"%s, hostnqn \"%s\"\n",
335	inv_data, data->hostnqn);
336	break;
337	case (offsetof(struct nvmf_connect_data, subsysnqn)):
338	dev_err(ctrl->device,
339	"%s, subsysnqn \"%s\"\n",
340	inv_data, data->subsysnqn);
341	break;
342	default:
343	dev_err(ctrl->device,
344	"%s, starting byte offset: %d\n",
345	inv_data, offset & 0xffff);
346	break;
347	}
348	} else {
349	char *inv_sqe = "Connect Invalid SQE Parameter";
350
351	switch (offset) {
352	case (offsetof(struct nvmf_connect_command, qid)):
353	dev_err(ctrl->device,
354	"%s, qid %d\n",
355	inv_sqe, cmd->connect.qid);
356	break;
357	default:
358	dev_err(ctrl->device,
359	"%s, starting byte offset: %d\n",
360	inv_sqe, offset);
361	}
362	}
363	break;
364	case NVME_SC_CONNECT_INVALID_HOST:
365	dev_err(ctrl->device,
366	"Connect for subsystem %s is not allowed, hostnqn: %s\n",
367	data->subsysnqn, data->hostnqn);
368	break;
369	case NVME_SC_CONNECT_CTRL_BUSY:
370	dev_err(ctrl->device,
371	"Connect command failed: controller is busy or not available\n");
372	break;
373	case NVME_SC_CONNECT_FORMAT:
374	dev_err(ctrl->device,
375	"Connect incompatible format: %d",
376	cmd->connect.recfmt);
377	break;
378	case NVME_SC_HOST_PATH_ERROR:
379	dev_err(ctrl->device,
380	"Connect command failed: host path error\n");
381	break;
382	case NVME_SC_AUTH_REQUIRED:
383	dev_err(ctrl->device,
384	"Connect command failed: authentication required\n");
385	break;
386	default:
387	dev_err(ctrl->device,
388	"Connect command failed, error wo/DNR bit: %d\n",
389	err_sctype);
390	break;
391	}
392	}
393
394	static struct nvmf_connect_data *nvmf_connect_data_prep(struct nvme_ctrl *ctrl,
395	u16 cntlid)
396	{
397	struct nvmf_connect_data *data;
398
399	data = kzalloc(sizeof(*data), GFP_KERNEL);
400	if (!data)
401	return NULL;
402
403	uuid_copy(dst: &data->hostid, src: &ctrl->opts->host->id);
404	data->cntlid = cpu_to_le16(cntlid);
405	strscpy(data->subsysnqn, ctrl->opts->subsysnqn, NVMF_NQN_SIZE);
406	strscpy(data->hostnqn, ctrl->opts->host->nqn, NVMF_NQN_SIZE);
407
408	return data;
409	}
410
411	static void nvmf_connect_cmd_prep(struct nvme_ctrl *ctrl, u16 qid,
412	struct nvme_command *cmd)
413	{
414	cmd->connect.opcode = nvme_fabrics_command;
415	cmd->connect.fctype = nvme_fabrics_type_connect;
416	cmd->connect.qid = cpu_to_le16(qid);
417
418	if (qid) {
419	cmd->connect.sqsize = cpu_to_le16(ctrl->sqsize);
420	} else {
421	cmd->connect.sqsize = cpu_to_le16(NVME_AQ_DEPTH - 1);
422
423	/*
424	* set keep-alive timeout in seconds granularity (ms * 1000)
425	*/
426	cmd->connect.kato = cpu_to_le32(ctrl->kato * 1000);
427	}
428
429	if (ctrl->opts->disable_sqflow)
430	cmd->connect.cattr |= NVME_CONNECT_DISABLE_SQFLOW;
431	}
432
433	/**
434	* nvmf_connect_admin_queue() - NVMe Fabrics Admin Queue "Connect"
435	* API function.
436	* @ctrl: Host nvme controller instance used to request
437	* a new NVMe controller allocation on the target
438	* system and establish an NVMe Admin connection to
439	* that controller.
440	*
441	* This function enables an NVMe host device to request a new allocation of
442	* an NVMe controller resource on a target system as well establish a
443	* fabrics-protocol connection of the NVMe Admin queue between the
444	* host system device and the allocated NVMe controller on the
445	* target system via a NVMe Fabrics "Connect" command.
446	*/
447	int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)
448	{
449	struct nvme_command cmd = { };
450	union nvme_result res;
451	struct nvmf_connect_data *data;
452	int ret;
453	u32 result;
454
455	nvmf_connect_cmd_prep(ctrl, qid: 0, cmd: &cmd);
456
457	data = nvmf_connect_data_prep(ctrl, cntlid: 0xffff);
458	if (!data)
459	return -ENOMEM;
460
461	ret = __nvme_submit_sync_cmd(q: ctrl->fabrics_q, cmd: &cmd, result: &res,
462	buffer: data, bufflen: sizeof(*data), NVME_QID_ANY,
463	flags: NVME_SUBMIT_AT_HEAD |
464	NVME_SUBMIT_NOWAIT |
465	NVME_SUBMIT_RESERVED);
466	if (ret) {
467	nvmf_log_connect_error(ctrl, errval: ret, le32_to_cpu(res.u32),
468	cmd: &cmd, data);
469	goto out_free_data;
470	}
471
472	result = le32_to_cpu(res.u32);
473	ctrl->cntlid = result & 0xFFFF;
474	if (result & (NVME_CONNECT_AUTHREQ_ATR | NVME_CONNECT_AUTHREQ_ASCR)) {
475	/* Check for secure concatenation */
476	if ((result & NVME_CONNECT_AUTHREQ_ASCR) &&
477	!ctrl->opts->concat) {
478	dev_warn(ctrl->device,
479	"qid 0: secure concatenation is not supported\n");
480	ret = -EOPNOTSUPP;
481	goto out_free_data;
482	}
483	/* Authentication required */
484	ret = nvme_auth_negotiate(ctrl, qid: 0);
485	if (ret) {
486	dev_warn(ctrl->device,
487	"qid 0: authentication setup failed\n");
488	goto out_free_data;
489	}
490	ret = nvme_auth_wait(ctrl, qid: 0);
491	if (ret) {
492	dev_warn(ctrl->device,
493	"qid 0: authentication failed, error %d\n",
494	ret);
495	} else
496	dev_info(ctrl->device,
497	"qid 0: authenticated\n");
498	}
499	out_free_data:
500	kfree(objp: data);
501	return ret;
502	}
503	EXPORT_SYMBOL_GPL(nvmf_connect_admin_queue);
504
505	/**
506	* nvmf_connect_io_queue() - NVMe Fabrics I/O Queue "Connect"
507	* API function.
508	* @ctrl: Host nvme controller instance used to establish an
509	* NVMe I/O queue connection to the already allocated NVMe
510	* controller on the target system.
511	* @qid: NVMe I/O queue number for the new I/O connection between
512	* host and target (note qid == 0 is illegal as this is
513	* the Admin queue, per NVMe standard).
514	*
515	* This function issues a fabrics-protocol connection
516	* of a NVMe I/O queue (via NVMe Fabrics "Connect" command)
517	* between the host system device and the allocated NVMe controller
518	* on the target system.
519	*
520	* Return:
521	* 0: success
522	* > 0: NVMe error status code
523	* < 0: Linux errno error code
524	*/
525	int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid)
526	{
527	struct nvme_command cmd = { };
528	struct nvmf_connect_data *data;
529	union nvme_result res;
530	int ret;
531	u32 result;
532
533	nvmf_connect_cmd_prep(ctrl, qid, cmd: &cmd);
534
535	data = nvmf_connect_data_prep(ctrl, cntlid: ctrl->cntlid);
536	if (!data)
537	return -ENOMEM;
538
539	ret = __nvme_submit_sync_cmd(q: ctrl->connect_q, cmd: &cmd, result: &res,
540	buffer: data, bufflen: sizeof(*data), qid,
541	flags: NVME_SUBMIT_AT_HEAD |
542	NVME_SUBMIT_RESERVED |
543	NVME_SUBMIT_NOWAIT);
544	if (ret) {
545	nvmf_log_connect_error(ctrl, errval: ret, le32_to_cpu(res.u32),
546	cmd: &cmd, data);
547	goto out_free_data;
548	}
549	result = le32_to_cpu(res.u32);
550	if (result & (NVME_CONNECT_AUTHREQ_ATR | NVME_CONNECT_AUTHREQ_ASCR)) {
551	/* Secure concatenation is not implemented */
552	if (result & NVME_CONNECT_AUTHREQ_ASCR) {
553	dev_warn(ctrl->device,
554	"qid %d: secure concatenation is not supported\n", qid);
555	ret = -EOPNOTSUPP;
556	goto out_free_data;
557	}
558	/* Authentication required */
559	ret = nvme_auth_negotiate(ctrl, qid);
560	if (ret) {
561	dev_warn(ctrl->device,
562	"qid %d: authentication setup failed\n", qid);
563	goto out_free_data;
564	}
565	ret = nvme_auth_wait(ctrl, qid);
566	if (ret) {
567	dev_warn(ctrl->device,
568	"qid %u: authentication failed, error %d\n",
569	qid, ret);
570	}
571	}
572	out_free_data:
573	kfree(objp: data);
574	return ret;
575	}
576	EXPORT_SYMBOL_GPL(nvmf_connect_io_queue);
577
578	/*
579	* Evaluate the status information returned by the transport in order to decided
580	* if a reconnect attempt should be scheduled.
581	*
582	* Do not retry when:
583	*
584	* - the DNR bit is set and the specification states no further connect
585	* attempts with the same set of parameters should be attempted.
586	*
587	* - when the authentication attempt fails, because the key was invalid.
588	* This error code is set on the host side.
589	*/
590	bool nvmf_should_reconnect(struct nvme_ctrl *ctrl, int status)
591	{
592	if (status > 0 && (status & NVME_STATUS_DNR))
593	return false;
594
595	if (status == -EKEYREJECTED || status == -ENOKEY)
596	return false;
597
598	if (ctrl->opts->max_reconnects == -1 ||
599	ctrl->nr_reconnects < ctrl->opts->max_reconnects)
600	return true;
601
602	return false;
603	}
604	EXPORT_SYMBOL_GPL(nvmf_should_reconnect);
605
606	/**
607	* nvmf_register_transport() - NVMe Fabrics Library registration function.
608	* @ops: Transport ops instance to be registered to the
609	* common fabrics library.
610	*
611	* API function that registers the type of specific transport fabric
612	* being implemented to the common NVMe fabrics library. Part of
613	* the overall init sequence of starting up a fabrics driver.
614	*/
615	int nvmf_register_transport(struct nvmf_transport_ops *ops)
616	{
617	if (!ops->create_ctrl)
618	return -EINVAL;
619
620	down_write(sem: &nvmf_transports_rwsem);
621	list_add_tail(new: &ops->entry, head: &nvmf_transports);
622	up_write(sem: &nvmf_transports_rwsem);
623
624	return 0;
625	}
626	EXPORT_SYMBOL_GPL(nvmf_register_transport);
627
628	/**
629	* nvmf_unregister_transport() - NVMe Fabrics Library unregistration function.
630	* @ops: Transport ops instance to be unregistered from the
631	* common fabrics library.
632	*
633	* Fabrics API function that unregisters the type of specific transport
634	* fabric being implemented from the common NVMe fabrics library.
635	* Part of the overall exit sequence of unloading the implemented driver.
636	*/
637	void nvmf_unregister_transport(struct nvmf_transport_ops *ops)
638	{
639	down_write(sem: &nvmf_transports_rwsem);
640	list_del(entry: &ops->entry);
641	up_write(sem: &nvmf_transports_rwsem);
642	}
643	EXPORT_SYMBOL_GPL(nvmf_unregister_transport);
644
645	static struct nvmf_transport_ops *nvmf_lookup_transport(
646	struct nvmf_ctrl_options *opts)
647	{
648	struct nvmf_transport_ops *ops;
649
650	lockdep_assert_held(&nvmf_transports_rwsem);
651
652	list_for_each_entry(ops, &nvmf_transports, entry) {
653	if (strcmp(ops->name, opts->transport) == 0)
654	return ops;
655	}
656
657	return NULL;
658	}
659
660	static struct key *nvmf_parse_key(int key_id)
661	{
662	struct key *key;
663
664	if (!IS_ENABLED(CONFIG_NVME_TCP_TLS)) {
665	pr_err("TLS is not supported\n");
666	return ERR_PTR(error: -EINVAL);
667	}
668
669	key = nvme_tls_key_lookup(key_id);
670	if (IS_ERR(ptr: key))
671	pr_err("key id %08x not found\n", key_id);
672	else
673	pr_debug("Using key id %08x\n", key_id);
674	return key;
675	}
676
677	static const match_table_t opt_tokens = {
678	{ NVMF_OPT_TRANSPORT, "transport=%s" },
679	{ NVMF_OPT_TRADDR, "traddr=%s" },
680	{ NVMF_OPT_TRSVCID, "trsvcid=%s" },
681	{ NVMF_OPT_NQN, "nqn=%s" },
682	{ NVMF_OPT_QUEUE_SIZE, "queue_size=%d" },
683	{ NVMF_OPT_NR_IO_QUEUES, "nr_io_queues=%d" },
684	{ NVMF_OPT_RECONNECT_DELAY, "reconnect_delay=%d" },
685	{ NVMF_OPT_CTRL_LOSS_TMO, "ctrl_loss_tmo=%d" },
686	{ NVMF_OPT_KATO, "keep_alive_tmo=%d" },
687	{ NVMF_OPT_HOSTNQN, "hostnqn=%s" },
688	{ NVMF_OPT_HOST_TRADDR, "host_traddr=%s" },
689	{ NVMF_OPT_HOST_IFACE, "host_iface=%s" },
690	{ NVMF_OPT_HOST_ID, "hostid=%s" },
691	{ NVMF_OPT_DUP_CONNECT, "duplicate_connect" },
692	{ NVMF_OPT_DISABLE_SQFLOW, "disable_sqflow" },
693	{ NVMF_OPT_HDR_DIGEST, "hdr_digest" },
694	{ NVMF_OPT_DATA_DIGEST, "data_digest" },
695	{ NVMF_OPT_NR_WRITE_QUEUES, "nr_write_queues=%d" },
696	{ NVMF_OPT_NR_POLL_QUEUES, "nr_poll_queues=%d" },
697	{ NVMF_OPT_TOS, "tos=%d" },
698	#ifdef CONFIG_NVME_TCP_TLS
699	{ NVMF_OPT_KEYRING, "keyring=%d" },
700	{ NVMF_OPT_TLS_KEY, "tls_key=%d" },
701	#endif
702	{ NVMF_OPT_FAIL_FAST_TMO, "fast_io_fail_tmo=%d" },
703	{ NVMF_OPT_DISCOVERY, "discovery" },
704	#ifdef CONFIG_NVME_HOST_AUTH
705	{ NVMF_OPT_DHCHAP_SECRET, "dhchap_secret=%s" },
706	{ NVMF_OPT_DHCHAP_CTRL_SECRET, "dhchap_ctrl_secret=%s" },
707	#endif
708	#ifdef CONFIG_NVME_TCP_TLS
709	{ NVMF_OPT_TLS, "tls" },
710	{ NVMF_OPT_CONCAT, "concat" },
711	#endif
712	{ NVMF_OPT_ERR, NULL }
713	};
714
715	static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
716	const char *buf)
717	{
718	substring_t args[MAX_OPT_ARGS];
719	char *options, *o, *p;
720	int token, ret = 0;
721	size_t nqnlen = 0;
722	int ctrl_loss_tmo = NVMF_DEF_CTRL_LOSS_TMO, key_id;
723	uuid_t hostid;
724	char hostnqn[NVMF_NQN_SIZE];
725	struct key *key;
726
727	/* Set defaults */
728	opts->queue_size = NVMF_DEF_QUEUE_SIZE;
729	opts->nr_io_queues = num_online_cpus();
730	opts->reconnect_delay = NVMF_DEF_RECONNECT_DELAY;
731	opts->kato = 0;
732	opts->duplicate_connect = false;
733	opts->fast_io_fail_tmo = NVMF_DEF_FAIL_FAST_TMO;
734	opts->hdr_digest = false;
735	opts->data_digest = false;
736	opts->tos = -1; /* < 0 == use transport default */
737	opts->tls = false;
738	opts->tls_key = NULL;
739	opts->keyring = NULL;
740	opts->concat = false;
741
742	options = o = kstrdup(s: buf, GFP_KERNEL);
743	if (!options)
744	return -ENOMEM;
745
746	/* use default host if not given by user space */
747	uuid_copy(dst: &hostid, src: &nvmf_default_host->id);
748	strscpy(hostnqn, nvmf_default_host->nqn, NVMF_NQN_SIZE);
749
750	while ((p = strsep(&o, ",\n")) != NULL) {
751	if (!*p)
752	continue;
753
754	token = match_token(p, table: opt_tokens, args);
755	opts->mask |= token;
756	switch (token) {
757	case NVMF_OPT_TRANSPORT:
758	p = match_strdup(args);
759	if (!p) {
760	ret = -ENOMEM;
761	goto out;
762	}
763	kfree(objp: opts->transport);
764	opts->transport = p;
765	break;
766	case NVMF_OPT_NQN:
767	p = match_strdup(args);
768	if (!p) {
769	ret = -ENOMEM;
770	goto out;
771	}
772	kfree(objp: opts->subsysnqn);
773	opts->subsysnqn = p;
774	nqnlen = strlen(opts->subsysnqn);
775	if (nqnlen >= NVMF_NQN_SIZE) {
776	pr_err("%s needs to be < %d bytes\n",
777	opts->subsysnqn, NVMF_NQN_SIZE);
778	ret = -EINVAL;
779	goto out;
780	}
781	opts->discovery_nqn =
782	!(strcmp(opts->subsysnqn,
783	NVME_DISC_SUBSYS_NAME));
784	break;
785	case NVMF_OPT_TRADDR:
786	p = match_strdup(args);
787	if (!p) {
788	ret = -ENOMEM;
789	goto out;
790	}
791	kfree(objp: opts->traddr);
792	opts->traddr = p;
793	break;
794	case NVMF_OPT_TRSVCID:
795	p = match_strdup(args);
796	if (!p) {
797	ret = -ENOMEM;
798	goto out;
799	}
800	kfree(objp: opts->trsvcid);
801	opts->trsvcid = p;
802	break;
803	case NVMF_OPT_QUEUE_SIZE:
804	if (match_int(args, result: &token)) {
805	ret = -EINVAL;
806	goto out;
807	}
808	if (token < NVMF_MIN_QUEUE_SIZE ||
809	token > NVMF_MAX_QUEUE_SIZE) {
810	pr_err("Invalid queue_size %d\n", token);
811	ret = -EINVAL;
812	goto out;
813	}
814	opts->queue_size = token;
815	break;
816	case NVMF_OPT_NR_IO_QUEUES:
817	if (match_int(args, result: &token)) {
818	ret = -EINVAL;
819	goto out;
820	}
821	if (token <= 0) {
822	pr_err("Invalid number of IOQs %d\n", token);
823	ret = -EINVAL;
824	goto out;
825	}
826	if (opts->discovery_nqn) {
827	pr_debug("Ignoring nr_io_queues value for discovery controller\n");
828	break;
829	}
830
831	opts->nr_io_queues = min_t(unsigned int,
832	num_online_cpus(), token);
833	break;
834	case NVMF_OPT_KATO:
835	if (match_int(args, result: &token)) {
836	ret = -EINVAL;
837	goto out;
838	}
839
840	if (token < 0) {
841	pr_err("Invalid keep_alive_tmo %d\n", token);
842	ret = -EINVAL;
843	goto out;
844	} else if (token == 0 && !opts->discovery_nqn) {
845	/* Allowed for debug */
846	pr_warn("keep_alive_tmo 0 won't execute keep alives!!!\n");
847	}
848	opts->kato = token;
849	break;
850	case NVMF_OPT_CTRL_LOSS_TMO:
851	if (match_int(args, result: &token)) {
852	ret = -EINVAL;
853	goto out;
854	}
855
856	if (token < 0)
857	pr_warn("ctrl_loss_tmo < 0 will reconnect forever\n");
858	ctrl_loss_tmo = token;
859	break;
860	case NVMF_OPT_FAIL_FAST_TMO:
861	if (match_int(args, result: &token)) {
862	ret = -EINVAL;
863	goto out;
864	}
865
866	if (token >= 0)
867	pr_warn("I/O fail on reconnect controller after %d sec\n",
868	token);
869	else
870	token = -1;
871
872	opts->fast_io_fail_tmo = token;
873	break;
874	case NVMF_OPT_HOSTNQN:
875	if (opts->host) {
876	pr_err("hostnqn already user-assigned: %s\n",
877	opts->host->nqn);
878	ret = -EADDRINUSE;
879	goto out;
880	}
881	p = match_strdup(args);
882	if (!p) {
883	ret = -ENOMEM;
884	goto out;
885	}
886	nqnlen = strlen(p);
887	if (nqnlen >= NVMF_NQN_SIZE) {
888	pr_err("%s needs to be < %d bytes\n",
889	p, NVMF_NQN_SIZE);
890	kfree(objp: p);
891	ret = -EINVAL;
892	goto out;
893	}
894	strscpy(hostnqn, p, NVMF_NQN_SIZE);
895	kfree(objp: p);
896	break;
897	case NVMF_OPT_RECONNECT_DELAY:
898	if (match_int(args, result: &token)) {
899	ret = -EINVAL;
900	goto out;
901	}
902	if (token <= 0) {
903	pr_err("Invalid reconnect_delay %d\n", token);
904	ret = -EINVAL;
905	goto out;
906	}
907	opts->reconnect_delay = token;
908	break;
909	case NVMF_OPT_HOST_TRADDR:
910	p = match_strdup(args);
911	if (!p) {
912	ret = -ENOMEM;
913	goto out;
914	}
915	kfree(objp: opts->host_traddr);
916	opts->host_traddr = p;
917	break;
918	case NVMF_OPT_HOST_IFACE:
919	p = match_strdup(args);
920	if (!p) {
921	ret = -ENOMEM;
922	goto out;
923	}
924	kfree(objp: opts->host_iface);
925	opts->host_iface = p;
926	break;
927	case NVMF_OPT_HOST_ID:
928	p = match_strdup(args);
929	if (!p) {
930	ret = -ENOMEM;
931	goto out;
932	}
933	ret = uuid_parse(uuid: p, u: &hostid);
934	if (ret) {
935	pr_err("Invalid hostid %s\n", p);
936	ret = -EINVAL;
937	kfree(objp: p);
938	goto out;
939	}
940	kfree(objp: p);
941	break;
942	case NVMF_OPT_DUP_CONNECT:
943	opts->duplicate_connect = true;
944	break;
945	case NVMF_OPT_DISABLE_SQFLOW:
946	opts->disable_sqflow = true;
947	break;
948	case NVMF_OPT_HDR_DIGEST:
949	opts->hdr_digest = true;
950	break;
951	case NVMF_OPT_DATA_DIGEST:
952	opts->data_digest = true;
953	break;
954	case NVMF_OPT_NR_WRITE_QUEUES:
955	if (match_int(args, result: &token)) {
956	ret = -EINVAL;
957	goto out;
958	}
959	if (token <= 0) {
960	pr_err("Invalid nr_write_queues %d\n", token);
961	ret = -EINVAL;
962	goto out;
963	}
964	opts->nr_write_queues = token;
965	break;
966	case NVMF_OPT_NR_POLL_QUEUES:
967	if (match_int(args, result: &token)) {
968	ret = -EINVAL;
969	goto out;
970	}
971	if (token <= 0) {
972	pr_err("Invalid nr_poll_queues %d\n", token);
973	ret = -EINVAL;
974	goto out;
975	}
976	opts->nr_poll_queues = token;
977	break;
978	case NVMF_OPT_TOS:
979	if (match_int(args, result: &token)) {
980	ret = -EINVAL;
981	goto out;
982	}
983	if (token < 0) {
984	pr_err("Invalid type of service %d\n", token);
985	ret = -EINVAL;
986	goto out;
987	}
988	if (token > 255) {
989	pr_warn("Clamping type of service to 255\n");
990	token = 255;
991	}
992	opts->tos = token;
993	break;
994	case NVMF_OPT_KEYRING:
995	if (match_int(args, result: &key_id) || key_id <= 0) {
996	ret = -EINVAL;
997	goto out;
998	}
999	key = nvmf_parse_key(key_id);
1000	if (IS_ERR(ptr: key)) {
1001	ret = PTR_ERR(ptr: key);
1002	goto out;
1003	}
1004	key_put(key: opts->keyring);
1005	opts->keyring = key;
1006	break;
1007	case NVMF_OPT_TLS_KEY:
1008	if (match_int(args, result: &key_id) || key_id <= 0) {
1009	ret = -EINVAL;
1010	goto out;
1011	}
1012	key = nvmf_parse_key(key_id);
1013	if (IS_ERR(ptr: key)) {
1014	ret = PTR_ERR(ptr: key);
1015	goto out;
1016	}
1017	key_put(key: opts->tls_key);
1018	opts->tls_key = key;
1019	break;
1020	case NVMF_OPT_DISCOVERY:
1021	opts->discovery_nqn = true;
1022	break;
1023	case NVMF_OPT_DHCHAP_SECRET:
1024	p = match_strdup(args);
1025	if (!p) {
1026	ret = -ENOMEM;
1027	goto out;
1028	}
1029	if (strlen(p) < 11 || strncmp(p, "DHHC-1:", 7)) {
1030	pr_err("Invalid DH-CHAP secret %s\n", p);
1031	ret = -EINVAL;
1032	goto out;
1033	}
1034	kfree(objp: opts->dhchap_secret);
1035	opts->dhchap_secret = p;
1036	break;
1037	case NVMF_OPT_DHCHAP_CTRL_SECRET:
1038	p = match_strdup(args);
1039	if (!p) {
1040	ret = -ENOMEM;
1041	goto out;
1042	}
1043	if (strlen(p) < 11 || strncmp(p, "DHHC-1:", 7)) {
1044	pr_err("Invalid DH-CHAP secret %s\n", p);
1045	ret = -EINVAL;
1046	goto out;
1047	}
1048	kfree(objp: opts->dhchap_ctrl_secret);
1049	opts->dhchap_ctrl_secret = p;
1050	break;
1051	case NVMF_OPT_TLS:
1052	if (!IS_ENABLED(CONFIG_NVME_TCP_TLS)) {
1053	pr_err("TLS is not supported\n");
1054	ret = -EINVAL;
1055	goto out;
1056	}
1057	opts->tls = true;
1058	break;
1059	case NVMF_OPT_CONCAT:
1060	if (!IS_ENABLED(CONFIG_NVME_TCP_TLS)) {
1061	pr_err("TLS is not supported\n");
1062	ret = -EINVAL;
1063	goto out;
1064	}
1065	opts->concat = true;
1066	break;
1067	default:
1068	pr_warn("unknown parameter or missing value '%s' in ctrl creation request\n",
1069	p);
1070	ret = -EINVAL;
1071	goto out;
1072	}
1073	}
1074
1075	if (opts->discovery_nqn) {
1076	opts->nr_io_queues = 0;
1077	opts->nr_write_queues = 0;
1078	opts->nr_poll_queues = 0;
1079	opts->duplicate_connect = true;
1080	} else {
1081	if (!opts->kato)
1082	opts->kato = NVME_DEFAULT_KATO;
1083	}
1084	if (ctrl_loss_tmo < 0) {
1085	opts->max_reconnects = -1;
1086	} else {
1087	opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
1088	opts->reconnect_delay);
1089	if (ctrl_loss_tmo < opts->fast_io_fail_tmo)
1090	pr_warn("failfast tmo (%d) larger than controller loss tmo (%d)\n",
1091	opts->fast_io_fail_tmo, ctrl_loss_tmo);
1092	}
1093	if (opts->concat) {
1094	if (opts->tls) {
1095	pr_err("Secure concatenation over TLS is not supported\n");
1096	ret = -EINVAL;
1097	goto out;
1098	}
1099	if (opts->tls_key) {
1100	pr_err("Cannot specify a TLS key for secure concatenation\n");
1101	ret = -EINVAL;
1102	goto out;
1103	}
1104	if (!opts->dhchap_secret) {
1105	pr_err("Need to enable DH-CHAP for secure concatenation\n");
1106	ret = -EINVAL;
1107	goto out;
1108	}
1109	}
1110
1111	opts->host = nvmf_host_add(hostnqn, id: &hostid);
1112	if (IS_ERR(ptr: opts->host)) {
1113	ret = PTR_ERR(ptr: opts->host);
1114	opts->host = NULL;
1115	goto out;
1116	}
1117
1118	out:
1119	kfree(objp: options);
1120	return ret;
1121	}
1122
1123	void nvmf_set_io_queues(struct nvmf_ctrl_options *opts, u32 nr_io_queues,
1124	u32 io_queues[HCTX_MAX_TYPES])
1125	{
1126	if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) {
1127	/*
1128	* separate read/write queues
1129	* hand out dedicated default queues only after we have
1130	* sufficient read queues.
1131	*/
1132	io_queues[HCTX_TYPE_READ] = opts->nr_io_queues;
1133	nr_io_queues -= io_queues[HCTX_TYPE_READ];
1134	io_queues[HCTX_TYPE_DEFAULT] =
1135	min(opts->nr_write_queues, nr_io_queues);
1136	nr_io_queues -= io_queues[HCTX_TYPE_DEFAULT];
1137	} else {
1138	/*
1139	* shared read/write queues
1140	* either no write queues were requested, or we don't have
1141	* sufficient queue count to have dedicated default queues.
1142	*/
1143	io_queues[HCTX_TYPE_DEFAULT] =
1144	min(opts->nr_io_queues, nr_io_queues);
1145	nr_io_queues -= io_queues[HCTX_TYPE_DEFAULT];
1146	}
1147
1148	if (opts->nr_poll_queues && nr_io_queues) {
1149	/* map dedicated poll queues only if we have queues left */
1150	io_queues[HCTX_TYPE_POLL] =
1151	min(opts->nr_poll_queues, nr_io_queues);
1152	}
1153	}
1154	EXPORT_SYMBOL_GPL(nvmf_set_io_queues);
1155
1156	void nvmf_map_queues(struct blk_mq_tag_set *set, struct nvme_ctrl *ctrl,
1157	u32 io_queues[HCTX_MAX_TYPES])
1158	{
1159	struct nvmf_ctrl_options *opts = ctrl->opts;
1160
1161	if (opts->nr_write_queues && io_queues[HCTX_TYPE_READ]) {
1162	/* separate read/write queues */
1163	set->map[HCTX_TYPE_DEFAULT].nr_queues =
1164	io_queues[HCTX_TYPE_DEFAULT];
1165	set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
1166	set->map[HCTX_TYPE_READ].nr_queues =
1167	io_queues[HCTX_TYPE_READ];
1168	set->map[HCTX_TYPE_READ].queue_offset =
1169	io_queues[HCTX_TYPE_DEFAULT];
1170	} else {
1171	/* shared read/write queues */
1172	set->map[HCTX_TYPE_DEFAULT].nr_queues =
1173	io_queues[HCTX_TYPE_DEFAULT];
1174	set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
1175	set->map[HCTX_TYPE_READ].nr_queues =
1176	io_queues[HCTX_TYPE_DEFAULT];
1177	set->map[HCTX_TYPE_READ].queue_offset = 0;
1178	}
1179
1180	blk_mq_map_queues(qmap: &set->map[HCTX_TYPE_DEFAULT]);
1181	blk_mq_map_queues(qmap: &set->map[HCTX_TYPE_READ]);
1182	if (opts->nr_poll_queues && io_queues[HCTX_TYPE_POLL]) {
1183	/* map dedicated poll queues only if we have queues left */
1184	set->map[HCTX_TYPE_POLL].nr_queues = io_queues[HCTX_TYPE_POLL];
1185	set->map[HCTX_TYPE_POLL].queue_offset =
1186	io_queues[HCTX_TYPE_DEFAULT] +
1187	io_queues[HCTX_TYPE_READ];
1188	blk_mq_map_queues(qmap: &set->map[HCTX_TYPE_POLL]);
1189	}
1190
1191	dev_info(ctrl->device,
1192	"mapped %d/%d/%d default/read/poll queues.\n",
1193	io_queues[HCTX_TYPE_DEFAULT],
1194	io_queues[HCTX_TYPE_READ],
1195	io_queues[HCTX_TYPE_POLL]);
1196	}
1197	EXPORT_SYMBOL_GPL(nvmf_map_queues);
1198
1199	static int nvmf_check_required_opts(struct nvmf_ctrl_options *opts,
1200	unsigned int required_opts)
1201	{
1202	if ((opts->mask & required_opts) != required_opts) {
1203	unsigned int i;
1204
1205	for (i = 0; i < ARRAY_SIZE(opt_tokens); i++) {
1206	if ((opt_tokens[i].token & required_opts) &&
1207	!(opt_tokens[i].token & opts->mask)) {
1208	pr_warn("missing parameter '%s'\n",
1209	opt_tokens[i].pattern);
1210	}
1211	}
1212
1213	return -EINVAL;
1214	}
1215
1216	return 0;
1217	}
1218
1219	bool nvmf_ip_options_match(struct nvme_ctrl *ctrl,
1220	struct nvmf_ctrl_options *opts)
1221	{
1222	if (!nvmf_ctlr_matches_baseopts(ctrl, opts) ||
1223	strcmp(opts->traddr, ctrl->opts->traddr) ||
1224	strcmp(opts->trsvcid, ctrl->opts->trsvcid))
1225	return false;
1226
1227	/*
1228	* Checking the local address or host interfaces is rough.
1229	*
1230	* In most cases, none is specified and the host port or
1231	* host interface is selected by the stack.
1232	*
1233	* Assume no match if:
1234	* - local address or host interface is specified and address
1235	* or host interface is not the same
1236	* - local address or host interface is not specified but
1237	* remote is, or vice versa (admin using specific
1238	* host_traddr/host_iface when it matters).
1239	*/
1240	if ((opts->mask & NVMF_OPT_HOST_TRADDR) &&
1241	(ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)) {
1242	if (strcmp(opts->host_traddr, ctrl->opts->host_traddr))
1243	return false;
1244	} else if ((opts->mask & NVMF_OPT_HOST_TRADDR) ||
1245	(ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)) {
1246	return false;
1247	}
1248
1249	if ((opts->mask & NVMF_OPT_HOST_IFACE) &&
1250	(ctrl->opts->mask & NVMF_OPT_HOST_IFACE)) {
1251	if (strcmp(opts->host_iface, ctrl->opts->host_iface))
1252	return false;
1253	} else if ((opts->mask & NVMF_OPT_HOST_IFACE) ||
1254	(ctrl->opts->mask & NVMF_OPT_HOST_IFACE)) {
1255	return false;
1256	}
1257
1258	return true;
1259	}
1260	EXPORT_SYMBOL_GPL(nvmf_ip_options_match);
1261
1262	static int nvmf_check_allowed_opts(struct nvmf_ctrl_options *opts,
1263	unsigned int allowed_opts)
1264	{
1265	if (opts->mask & ~allowed_opts) {
1266	unsigned int i;
1267
1268	for (i = 0; i < ARRAY_SIZE(opt_tokens); i++) {
1269	if ((opt_tokens[i].token & opts->mask) &&
1270	(opt_tokens[i].token & ~allowed_opts)) {
1271	pr_warn("invalid parameter '%s'\n",
1272	opt_tokens[i].pattern);
1273	}
1274	}
1275
1276	return -EINVAL;
1277	}
1278
1279	return 0;
1280	}
1281
1282	void nvmf_free_options(struct nvmf_ctrl_options *opts)
1283	{
1284	nvmf_host_put(host: opts->host);
1285	key_put(key: opts->keyring);
1286	key_put(key: opts->tls_key);
1287	kfree(objp: opts->transport);
1288	kfree(objp: opts->traddr);
1289	kfree(objp: opts->trsvcid);
1290	kfree(objp: opts->subsysnqn);
1291	kfree(objp: opts->host_traddr);
1292	kfree(objp: opts->host_iface);
1293	kfree(objp: opts->dhchap_secret);
1294	kfree(objp: opts->dhchap_ctrl_secret);
1295	kfree(objp: opts);
1296	}
1297	EXPORT_SYMBOL_GPL(nvmf_free_options);
1298
1299	#define NVMF_REQUIRED_OPTS (NVMF_OPT_TRANSPORT | NVMF_OPT_NQN)
1300	#define NVMF_ALLOWED_OPTS (NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \
1301	NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \
1302	NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT |\
1303	NVMF_OPT_DISABLE_SQFLOW | NVMF_OPT_DISCOVERY |\
1304	NVMF_OPT_FAIL_FAST_TMO | NVMF_OPT_DHCHAP_SECRET |\
1305	NVMF_OPT_DHCHAP_CTRL_SECRET)
1306
1307	static struct nvme_ctrl *
1308	nvmf_create_ctrl(struct device *dev, const char *buf)
1309	{
1310	struct nvmf_ctrl_options *opts;
1311	struct nvmf_transport_ops *ops;
1312	struct nvme_ctrl *ctrl;
1313	int ret;
1314
1315	opts = kzalloc(sizeof(*opts), GFP_KERNEL);
1316	if (!opts)
1317	return ERR_PTR(error: -ENOMEM);
1318
1319	ret = nvmf_parse_options(opts, buf);
1320	if (ret)
1321	goto out_free_opts;
1322
1323
1324	request_module("nvme-%s", opts->transport);
1325
1326	/*
1327	* Check the generic options first as we need a valid transport for
1328	* the lookup below. Then clear the generic flags so that transport
1329	* drivers don't have to care about them.
1330	*/
1331	ret = nvmf_check_required_opts(opts, NVMF_REQUIRED_OPTS);
1332	if (ret)
1333	goto out_free_opts;
1334	opts->mask &= ~NVMF_REQUIRED_OPTS;
1335
1336	down_read(sem: &nvmf_transports_rwsem);
1337	ops = nvmf_lookup_transport(opts);
1338	if (!ops) {
1339	pr_info("no handler found for transport %s.\n",
1340	opts->transport);
1341	ret = -EINVAL;
1342	goto out_unlock;
1343	}
1344
1345	if (!try_module_get(module: ops->module)) {
1346	ret = -EBUSY;
1347	goto out_unlock;
1348	}
1349	up_read(sem: &nvmf_transports_rwsem);
1350
1351	ret = nvmf_check_required_opts(opts, required_opts: ops->required_opts);
1352	if (ret)
1353	goto out_module_put;
1354	ret = nvmf_check_allowed_opts(opts, NVMF_ALLOWED_OPTS |
1355	ops->allowed_opts | ops->required_opts);
1356	if (ret)
1357	goto out_module_put;
1358
1359	ctrl = ops->create_ctrl(dev, opts);
1360	if (IS_ERR(ptr: ctrl)) {
1361	ret = PTR_ERR(ptr: ctrl);
1362	goto out_module_put;
1363	}
1364
1365	module_put(module: ops->module);
1366	return ctrl;
1367
1368	out_module_put:
1369	module_put(module: ops->module);
1370	goto out_free_opts;
1371	out_unlock:
1372	up_read(sem: &nvmf_transports_rwsem);
1373	out_free_opts:
1374	nvmf_free_options(opts);
1375	return ERR_PTR(error: ret);
1376	}
1377
1378	static const struct class nvmf_class = {
1379	.name = "nvme-fabrics",
1380	};
1381
1382	static struct device *nvmf_device;
1383	static DEFINE_MUTEX(nvmf_dev_mutex);
1384
1385	static ssize_t nvmf_dev_write(struct file *file, const char __user *ubuf,
1386	size_t count, loff_t *pos)
1387	{
1388	struct seq_file *seq_file = file->private_data;
1389	struct nvme_ctrl *ctrl;
1390	const char *buf;
1391	int ret = 0;
1392
1393	if (count > PAGE_SIZE)
1394	return -ENOMEM;
1395
1396	buf = memdup_user_nul(ubuf, count);
1397	if (IS_ERR(ptr: buf))
1398	return PTR_ERR(ptr: buf);
1399
1400	mutex_lock(&nvmf_dev_mutex);
1401	if (seq_file->private) {
1402	ret = -EINVAL;
1403	goto out_unlock;
1404	}
1405
1406	ctrl = nvmf_create_ctrl(dev: nvmf_device, buf);
1407	if (IS_ERR(ptr: ctrl)) {
1408	ret = PTR_ERR(ptr: ctrl);
1409	goto out_unlock;
1410	}
1411
1412	seq_file->private = ctrl;
1413
1414	out_unlock:
1415	mutex_unlock(lock: &nvmf_dev_mutex);
1416	kfree(objp: buf);
1417	return ret ? ret : count;
1418	}
1419
1420	static void __nvmf_concat_opt_tokens(struct seq_file *seq_file)
1421	{
1422	const struct match_token *tok;
1423	int idx;
1424
1425	/*
1426	* Add dummy entries for instance and cntlid to
1427	* signal an invalid/non-existing controller
1428	*/
1429	seq_puts(m: seq_file, s: "instance=-1,cntlid=-1");
1430	for (idx = 0; idx < ARRAY_SIZE(opt_tokens); idx++) {
1431	tok = &opt_tokens[idx];
1432	if (tok->token == NVMF_OPT_ERR)
1433	continue;
1434	seq_putc(m: seq_file, c: ',');
1435	seq_puts(m: seq_file, s: tok->pattern);
1436	}
1437	seq_putc(m: seq_file, c: '\n');
1438	}
1439
1440	static int nvmf_dev_show(struct seq_file *seq_file, void *private)
1441	{
1442	struct nvme_ctrl *ctrl;
1443
1444	mutex_lock(&nvmf_dev_mutex);
1445	ctrl = seq_file->private;
1446	if (!ctrl) {
1447	__nvmf_concat_opt_tokens(seq_file);
1448	goto out_unlock;
1449	}
1450
1451	seq_printf(m: seq_file, fmt: "instance=%d,cntlid=%d\n",
1452	ctrl->instance, ctrl->cntlid);
1453
1454	out_unlock:
1455	mutex_unlock(lock: &nvmf_dev_mutex);
1456	return 0;
1457	}
1458
1459	static int nvmf_dev_open(struct inode *inode, struct file *file)
1460	{
1461	/*
1462	* The miscdevice code initializes file->private_data, but doesn't
1463	* make use of it later.
1464	*/
1465	file->private_data = NULL;
1466	return single_open(file, nvmf_dev_show, NULL);
1467	}
1468
1469	static int nvmf_dev_release(struct inode *inode, struct file *file)
1470	{
1471	struct seq_file *seq_file = file->private_data;
1472	struct nvme_ctrl *ctrl = seq_file->private;
1473
1474	if (ctrl)
1475	nvme_put_ctrl(ctrl);
1476	return single_release(inode, file);
1477	}
1478
1479	static const struct file_operations nvmf_dev_fops = {
1480	.owner = THIS_MODULE,
1481	.write = nvmf_dev_write,
1482	.read = seq_read,
1483	.open = nvmf_dev_open,
1484	.release = nvmf_dev_release,
1485	};
1486
1487	static struct miscdevice nvmf_misc = {
1488	.minor = MISC_DYNAMIC_MINOR,
1489	.name = "nvme-fabrics",
1490	.fops = &nvmf_dev_fops,
1491	};
1492
1493	static int __init nvmf_init(void)
1494	{
1495	int ret;
1496
1497	nvmf_default_host = nvmf_host_default();
1498	if (!nvmf_default_host)
1499	return -ENOMEM;
1500
1501	ret = class_register(class: &nvmf_class);
1502	if (ret) {
1503	pr_err("couldn't register class nvme-fabrics\n");
1504	goto out_free_host;
1505	}
1506
1507	nvmf_device =
1508	device_create(cls: &nvmf_class, NULL, MKDEV(0, 0), NULL, fmt: "ctl");
1509	if (IS_ERR(ptr: nvmf_device)) {
1510	pr_err("couldn't create nvme-fabrics device!\n");
1511	ret = PTR_ERR(ptr: nvmf_device);
1512	goto out_destroy_class;
1513	}
1514
1515	ret = misc_register(misc: &nvmf_misc);
1516	if (ret) {
1517	pr_err("couldn't register misc device: %d\n", ret);
1518	goto out_destroy_device;
1519	}
1520
1521	return 0;
1522
1523	out_destroy_device:
1524	device_destroy(cls: &nvmf_class, MKDEV(0, 0));
1525	out_destroy_class:
1526	class_unregister(class: &nvmf_class);
1527	out_free_host:
1528	nvmf_host_put(host: nvmf_default_host);
1529	return ret;
1530	}
1531
1532	static void __exit nvmf_exit(void)
1533	{
1534	misc_deregister(misc: &nvmf_misc);
1535	device_destroy(cls: &nvmf_class, MKDEV(0, 0));
1536	class_unregister(class: &nvmf_class);
1537	nvmf_host_put(host: nvmf_default_host);
1538
1539	BUILD_BUG_ON(sizeof(struct nvmf_common_command) != 64);
1540	BUILD_BUG_ON(sizeof(struct nvmf_connect_command) != 64);
1541	BUILD_BUG_ON(sizeof(struct nvmf_property_get_command) != 64);
1542	BUILD_BUG_ON(sizeof(struct nvmf_property_set_command) != 64);
1543	BUILD_BUG_ON(sizeof(struct nvmf_auth_send_command) != 64);
1544	BUILD_BUG_ON(sizeof(struct nvmf_auth_receive_command) != 64);
1545	BUILD_BUG_ON(sizeof(struct nvmf_connect_data) != 1024);
1546	BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_negotiate_data) != 8);
1547	BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_challenge_data) != 16);
1548	BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_reply_data) != 16);
1549	BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_success1_data) != 16);
1550	BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_success2_data) != 16);
1551	}
1552
1553	MODULE_LICENSE("GPL v2");
1554	MODULE_DESCRIPTION("NVMe host fabrics library");
1555
1556	module_init(nvmf_init);
1557	module_exit(nvmf_exit);
1558