1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* vDPA bus.
4	*
5	* Copyright (c) 2020, Red Hat. All rights reserved.
6	* Author: Jason Wang <jasowang@redhat.com>
7	*
8	*/
9
10	#include <linux/module.h>
11	#include <linux/idr.h>
12	#include <linux/slab.h>
13	#include <linux/vdpa.h>
14	#include <uapi/linux/vdpa.h>
15	#include <net/genetlink.h>
16	#include <linux/mod_devicetable.h>
17	#include <linux/virtio_ids.h>
18
19	static LIST_HEAD(mdev_head);
20	/* A global mutex that protects vdpa management device and device level operations. */
21	static DECLARE_RWSEM(vdpa_dev_lock);
22	static DEFINE_IDA(vdpa_index_ida);
23
24	void vdpa_set_status(struct vdpa_device *vdev, u8 status)
25	{
26	down_write(sem: &vdev->cf_lock);
27	vdev->config->set_status(vdev, status);
28	up_write(sem: &vdev->cf_lock);
29	}
30	EXPORT_SYMBOL(vdpa_set_status);
31
32	static struct genl_family vdpa_nl_family;
33
34	static int vdpa_dev_probe(struct device *d)
35	{
36	struct vdpa_device *vdev = dev_to_vdpa(dev: d);
37	struct vdpa_driver *drv = drv_to_vdpa(driver: vdev->dev.driver);
38	const struct vdpa_config_ops *ops = vdev->config;
39	u32 max_num, min_num = 1;
40	int ret = 0;
41
42	d->dma_mask = &d->coherent_dma_mask;
43	ret = dma_set_mask_and_coherent(dev: d, DMA_BIT_MASK(64));
44	if (ret)
45	return ret;
46
47	max_num = ops->get_vq_num_max(vdev);
48	if (ops->get_vq_num_min)
49	min_num = ops->get_vq_num_min(vdev);
50	if (max_num < min_num)
51	return -EINVAL;
52
53	if (drv && drv->probe)
54	ret = drv->probe(vdev);
55
56	return ret;
57	}
58
59	static void vdpa_dev_remove(struct device *d)
60	{
61	struct vdpa_device *vdev = dev_to_vdpa(dev: d);
62	struct vdpa_driver *drv = drv_to_vdpa(driver: vdev->dev.driver);
63
64	if (drv && drv->remove)
65	drv->remove(vdev);
66	}
67
68	static int vdpa_dev_match(struct device *dev, const struct device_driver *drv)
69	{
70	struct vdpa_device *vdev = dev_to_vdpa(dev: dev);
71
72	/* Check override first, and if set, only use the named driver */
73	if (vdev->driver_override)
74	return strcmp(vdev->driver_override, drv->name) == 0;
75
76	/* Currently devices must be supported by all vDPA bus drivers */
77	return 1;
78	}
79
80	static ssize_t driver_override_store(struct device *dev,
81	struct device_attribute *attr,
82	const char *buf, size_t count)
83	{
84	struct vdpa_device *vdev = dev_to_vdpa(dev: dev);
85	int ret;
86
87	ret = driver_set_override(dev, override: &vdev->driver_override, s: buf, len: count);
88	if (ret)
89	return ret;
90
91	return count;
92	}
93
94	static ssize_t driver_override_show(struct device *dev,
95	struct device_attribute *attr, char *buf)
96	{
97	struct vdpa_device *vdev = dev_to_vdpa(dev: dev);
98	ssize_t len;
99
100	device_lock(dev);
101	len = sysfs_emit(buf, fmt: "%s\n", vdev->driver_override);
102	device_unlock(dev);
103
104	return len;
105	}
106	static DEVICE_ATTR_RW(driver_override);
107
108	static struct attribute *vdpa_dev_attrs[] = {
109	&dev_attr_driver_override.attr,
110	NULL,
111	};
112
113	static const struct attribute_group vdpa_dev_group = {
114	.attrs = vdpa_dev_attrs,
115	};
116	__ATTRIBUTE_GROUPS(vdpa_dev);
117
118	static const struct bus_type vdpa_bus = {
119	.name = "vdpa",
120	.dev_groups = vdpa_dev_groups,
121	.match = vdpa_dev_match,
122	.probe = vdpa_dev_probe,
123	.remove = vdpa_dev_remove,
124	};
125
126	static void vdpa_release_dev(struct device *d)
127	{
128	struct vdpa_device *vdev = dev_to_vdpa(dev: d);
129	const struct vdpa_config_ops *ops = vdev->config;
130
131	if (ops->free)
132	ops->free(vdev);
133
134	ida_free(&vdpa_index_ida, id: vdev->index);
135	kfree(objp: vdev->driver_override);
136	kfree(objp: vdev);
137	}
138
139	/**
140	* __vdpa_alloc_device - allocate and initilaize a vDPA device
141	* This allows driver to some prepartion after device is
142	* initialized but before registered.
143	* @parent: the parent device
144	* @config: the bus operations that is supported by this device
145	* @map: the map operations that is supported by this device
146	* @ngroups: number of groups supported by this device
147	* @nas: number of address spaces supported by this device
148	* @size: size of the parent structure that contains private data
149	* @name: name of the vdpa device; optional.
150	* @use_va: indicate whether virtual address must be used by this device
151	*
152	* Driver should use vdpa_alloc_device() wrapper macro instead of
153	* using this directly.
154	*
155	* Return: Returns an error when parent/config/map is not set or fail to get
156	* ida.
157	*/
158	struct vdpa_device *__vdpa_alloc_device(struct device *parent,
159	const struct vdpa_config_ops *config,
160	const struct virtio_map_ops *map,
161	unsigned int ngroups, unsigned int nas,
162	size_t size, const char *name,
163	bool use_va)
164	{
165	struct vdpa_device *vdev;
166	int err = -EINVAL;
167
168	if (!config)
169	goto err;
170
171	if (!!config->dma_map != !!config->dma_unmap)
172	goto err;
173
174	/* It should only work for the device that use on-chip IOMMU */
175	if (use_va && !(config->dma_map || config->set_map))
176	goto err;
177
178	err = -ENOMEM;
179	vdev = kzalloc(size, GFP_KERNEL);
180	if (!vdev)
181	goto err;
182
183	err = ida_alloc(ida: &vdpa_index_ida, GFP_KERNEL);
184	if (err < 0)
185	goto err_ida;
186
187	vdev->dev.bus = &vdpa_bus;
188	vdev->dev.parent = parent;
189	vdev->dev.release = vdpa_release_dev;
190	vdev->index = err;
191	vdev->config = config;
192	vdev->map = map;
193	vdev->features_valid = false;
194	vdev->use_va = use_va;
195	vdev->ngroups = ngroups;
196	vdev->nas = nas;
197
198	if (name)
199	err = dev_set_name(dev: &vdev->dev, name: "%s", name);
200	else
201	err = dev_set_name(dev: &vdev->dev, name: "vdpa%u", vdev->index);
202	if (err)
203	goto err_name;
204
205	init_rwsem(&vdev->cf_lock);
206	device_initialize(dev: &vdev->dev);
207
208	return vdev;
209
210	err_name:
211	ida_free(&vdpa_index_ida, id: vdev->index);
212	err_ida:
213	kfree(objp: vdev);
214	err:
215	return ERR_PTR(error: err);
216	}
217	EXPORT_SYMBOL_GPL(__vdpa_alloc_device);
218
219	static int vdpa_name_match(struct device *dev, const void *data)
220	{
221	struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev);
222
223	return (strcmp(dev_name(dev: &vdev->dev), data) == 0);
224	}
225
226	static int __vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
227	{
228	struct device *dev;
229
230	vdev->nvqs = nvqs;
231
232	lockdep_assert_held(&vdpa_dev_lock);
233	dev = bus_find_device(bus: &vdpa_bus, NULL, data: dev_name(dev: &vdev->dev), match: vdpa_name_match);
234	if (dev) {
235	put_device(dev);
236	return -EEXIST;
237	}
238	return device_add(dev: &vdev->dev);
239	}
240
241	/**
242	* _vdpa_register_device - register a vDPA device with vdpa lock held
243	* Caller must have a succeed call of vdpa_alloc_device() before.
244	* Caller must invoke this routine in the management device dev_add()
245	* callback after setting up valid mgmtdev for this vdpa device.
246	* @vdev: the vdpa device to be registered to vDPA bus
247	* @nvqs: number of virtqueues supported by this device
248	*
249	* Return: Returns an error when fail to add device to vDPA bus
250	*/
251	int _vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
252	{
253	if (!vdev->mdev)
254	return -EINVAL;
255
256	return __vdpa_register_device(vdev, nvqs);
257	}
258	EXPORT_SYMBOL_GPL(_vdpa_register_device);
259
260	/**
261	* vdpa_register_device - register a vDPA device
262	* Callers must have a succeed call of vdpa_alloc_device() before.
263	* @vdev: the vdpa device to be registered to vDPA bus
264	* @nvqs: number of virtqueues supported by this device
265	*
266	* Return: Returns an error when fail to add to vDPA bus
267	*/
268	int vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
269	{
270	int err;
271
272	down_write(sem: &vdpa_dev_lock);
273	err = __vdpa_register_device(vdev, nvqs);
274	up_write(sem: &vdpa_dev_lock);
275	return err;
276	}
277	EXPORT_SYMBOL_GPL(vdpa_register_device);
278
279	/**
280	* _vdpa_unregister_device - unregister a vDPA device
281	* Caller must invoke this routine as part of management device dev_del()
282	* callback.
283	* @vdev: the vdpa device to be unregisted from vDPA bus
284	*/
285	void _vdpa_unregister_device(struct vdpa_device *vdev)
286	{
287	lockdep_assert_held(&vdpa_dev_lock);
288	WARN_ON(!vdev->mdev);
289	device_unregister(dev: &vdev->dev);
290	}
291	EXPORT_SYMBOL_GPL(_vdpa_unregister_device);
292
293	/**
294	* vdpa_unregister_device - unregister a vDPA device
295	* @vdev: the vdpa device to be unregisted from vDPA bus
296	*/
297	void vdpa_unregister_device(struct vdpa_device *vdev)
298	{
299	down_write(sem: &vdpa_dev_lock);
300	device_unregister(dev: &vdev->dev);
301	up_write(sem: &vdpa_dev_lock);
302	}
303	EXPORT_SYMBOL_GPL(vdpa_unregister_device);
304
305	/**
306	* __vdpa_register_driver - register a vDPA device driver
307	* @drv: the vdpa device driver to be registered
308	* @owner: module owner of the driver
309	*
310	* Return: Returns an err when fail to do the registration
311	*/
312	int __vdpa_register_driver(struct vdpa_driver *drv, struct module *owner)
313	{
314	drv->driver.bus = &vdpa_bus;
315	drv->driver.owner = owner;
316
317	return driver_register(drv: &drv->driver);
318	}
319	EXPORT_SYMBOL_GPL(__vdpa_register_driver);
320
321	/**
322	* vdpa_unregister_driver - unregister a vDPA device driver
323	* @drv: the vdpa device driver to be unregistered
324	*/
325	void vdpa_unregister_driver(struct vdpa_driver *drv)
326	{
327	driver_unregister(drv: &drv->driver);
328	}
329	EXPORT_SYMBOL_GPL(vdpa_unregister_driver);
330
331	/**
332	* vdpa_mgmtdev_register - register a vdpa management device
333	*
334	* @mdev: Pointer to vdpa management device
335	* vdpa_mgmtdev_register() register a vdpa management device which supports
336	* vdpa device management.
337	* Return: Returns 0 on success or failure when required callback ops are not
338	* initialized.
339	*/
340	int vdpa_mgmtdev_register(struct vdpa_mgmt_dev *mdev)
341	{
342	if (!mdev->device || !mdev->ops || !mdev->ops->dev_add || !mdev->ops->dev_del)
343	return -EINVAL;
344
345	INIT_LIST_HEAD(list: &mdev->list);
346	down_write(sem: &vdpa_dev_lock);
347	list_add_tail(new: &mdev->list, head: &mdev_head);
348	up_write(sem: &vdpa_dev_lock);
349	return 0;
350	}
351	EXPORT_SYMBOL_GPL(vdpa_mgmtdev_register);
352
353	static int vdpa_match_remove(struct device *dev, void *data)
354	{
355	struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev);
356	struct vdpa_mgmt_dev *mdev = vdev->mdev;
357
358	if (mdev == data)
359	mdev->ops->dev_del(mdev, vdev);
360	return 0;
361	}
362
363	void vdpa_mgmtdev_unregister(struct vdpa_mgmt_dev *mdev)
364	{
365	down_write(sem: &vdpa_dev_lock);
366
367	list_del(entry: &mdev->list);
368
369	/* Filter out all the entries belong to this management device and delete it. */
370	bus_for_each_dev(bus: &vdpa_bus, NULL, data: mdev, fn: vdpa_match_remove);
371
372	up_write(sem: &vdpa_dev_lock);
373	}
374	EXPORT_SYMBOL_GPL(vdpa_mgmtdev_unregister);
375
376	static void vdpa_get_config_unlocked(struct vdpa_device *vdev,
377	unsigned int offset,
378	void *buf, unsigned int len)
379	{
380	const struct vdpa_config_ops *ops = vdev->config;
381
382	/*
383	* Config accesses aren't supposed to trigger before features are set.
384	* If it does happen we assume a legacy guest.
385	*/
386	if (!vdev->features_valid)
387	vdpa_set_features_unlocked(vdev, features: 0);
388	ops->get_config(vdev, offset, buf, len);
389	}
390
391	/**
392	* vdpa_get_config - Get one or more device configuration fields.
393	* @vdev: vdpa device to operate on
394	* @offset: starting byte offset of the field
395	* @buf: buffer pointer to read to
396	* @len: length of the configuration fields in bytes
397	*/
398	void vdpa_get_config(struct vdpa_device *vdev, unsigned int offset,
399	void *buf, unsigned int len)
400	{
401	down_read(sem: &vdev->cf_lock);
402	vdpa_get_config_unlocked(vdev, offset, buf, len);
403	up_read(sem: &vdev->cf_lock);
404	}
405	EXPORT_SYMBOL_GPL(vdpa_get_config);
406
407	/**
408	* vdpa_set_config - Set one or more device configuration fields.
409	* @vdev: vdpa device to operate on
410	* @offset: starting byte offset of the field
411	* @buf: buffer pointer to read from
412	* @length: length of the configuration fields in bytes
413	*/
414	void vdpa_set_config(struct vdpa_device *vdev, unsigned int offset,
415	const void *buf, unsigned int length)
416	{
417	down_write(sem: &vdev->cf_lock);
418	vdev->config->set_config(vdev, offset, buf, length);
419	up_write(sem: &vdev->cf_lock);
420	}
421	EXPORT_SYMBOL_GPL(vdpa_set_config);
422
423	static bool mgmtdev_handle_match(const struct vdpa_mgmt_dev *mdev,
424	const char *busname, const char *devname)
425	{
426	/* Bus name is optional for simulated management device, so ignore the
427	* device with bus if bus attribute is provided.
428	*/
429	if ((busname && !mdev->device->bus) || (!busname && mdev->device->bus))
430	return false;
431
432	if (!busname && strcmp(dev_name(dev: mdev->device), devname) == 0)
433	return true;
434
435	if (busname && (strcmp(mdev->device->bus->name, busname) == 0) &&
436	(strcmp(dev_name(dev: mdev->device), devname) == 0))
437	return true;
438
439	return false;
440	}
441
442	static struct vdpa_mgmt_dev *vdpa_mgmtdev_get_from_attr(struct nlattr **attrs)
443	{
444	struct vdpa_mgmt_dev *mdev;
445	const char *busname = NULL;
446	const char *devname;
447
448	if (!attrs[VDPA_ATTR_MGMTDEV_DEV_NAME])
449	return ERR_PTR(error: -EINVAL);
450	devname = nla_data(nla: attrs[VDPA_ATTR_MGMTDEV_DEV_NAME]);
451	if (attrs[VDPA_ATTR_MGMTDEV_BUS_NAME])
452	busname = nla_data(nla: attrs[VDPA_ATTR_MGMTDEV_BUS_NAME]);
453
454	list_for_each_entry(mdev, &mdev_head, list) {
455	if (mgmtdev_handle_match(mdev, busname, devname))
456	return mdev;
457	}
458	return ERR_PTR(error: -ENODEV);
459	}
460
461	static int vdpa_nl_mgmtdev_handle_fill(struct sk_buff *msg, const struct vdpa_mgmt_dev *mdev)
462	{
463	if (mdev->device->bus &&
464	nla_put_string(skb: msg, attrtype: VDPA_ATTR_MGMTDEV_BUS_NAME, str: mdev->device->bus->name))
465	return -EMSGSIZE;
466	if (nla_put_string(skb: msg, attrtype: VDPA_ATTR_MGMTDEV_DEV_NAME, str: dev_name(dev: mdev->device)))
467	return -EMSGSIZE;
468	return 0;
469	}
470
471	static u64 vdpa_mgmtdev_get_classes(const struct vdpa_mgmt_dev *mdev,
472	unsigned int *nclasses)
473	{
474	u64 supported_classes = 0;
475	unsigned int n = 0;
476
477	for (int i = 0; mdev->id_table[i].device; i++) {
478	if (mdev->id_table[i].device > 63)
479	continue;
480	supported_classes |= BIT_ULL(mdev->id_table[i].device);
481	n++;
482	}
483	if (nclasses)
484	*nclasses = n;
485
486	return supported_classes;
487	}
488
489	static int vdpa_mgmtdev_fill(const struct vdpa_mgmt_dev *mdev, struct sk_buff *msg,
490	u32 portid, u32 seq, int flags)
491	{
492	void *hdr;
493	int err;
494
495	hdr = genlmsg_put(skb: msg, portid, seq, family: &vdpa_nl_family, flags, cmd: VDPA_CMD_MGMTDEV_NEW);
496	if (!hdr)
497	return -EMSGSIZE;
498	err = vdpa_nl_mgmtdev_handle_fill(msg, mdev);
499	if (err)
500	goto msg_err;
501
502	if (nla_put_u64_64bit(skb: msg, attrtype: VDPA_ATTR_MGMTDEV_SUPPORTED_CLASSES,
503	value: vdpa_mgmtdev_get_classes(mdev, NULL),
504	padattr: VDPA_ATTR_UNSPEC)) {
505	err = -EMSGSIZE;
506	goto msg_err;
507	}
508	if (nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_MGMTDEV_MAX_VQS,
509	value: mdev->max_supported_vqs)) {
510	err = -EMSGSIZE;
511	goto msg_err;
512	}
513	if (nla_put_u64_64bit(skb: msg, attrtype: VDPA_ATTR_DEV_SUPPORTED_FEATURES,
514	value: mdev->supported_features, padattr: VDPA_ATTR_PAD)) {
515	err = -EMSGSIZE;
516	goto msg_err;
517	}
518
519	genlmsg_end(skb: msg, hdr);
520	return 0;
521
522	msg_err:
523	genlmsg_cancel(skb: msg, hdr);
524	return err;
525	}
526
527	static int vdpa_nl_cmd_mgmtdev_get_doit(struct sk_buff *skb, struct genl_info *info)
528	{
529	struct vdpa_mgmt_dev *mdev;
530	struct sk_buff *msg;
531	int err;
532
533	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
534	if (!msg)
535	return -ENOMEM;
536
537	down_read(sem: &vdpa_dev_lock);
538	mdev = vdpa_mgmtdev_get_from_attr(attrs: info->attrs);
539	if (IS_ERR(ptr: mdev)) {
540	up_read(sem: &vdpa_dev_lock);
541	NL_SET_ERR_MSG_MOD(info->extack, "Fail to find the specified mgmt device");
542	err = PTR_ERR(ptr: mdev);
543	goto out;
544	}
545
546	err = vdpa_mgmtdev_fill(mdev, msg, portid: info->snd_portid, seq: info->snd_seq, flags: 0);
547	up_read(sem: &vdpa_dev_lock);
548	if (err)
549	goto out;
550	err = genlmsg_reply(skb: msg, info);
551	return err;
552
553	out:
554	nlmsg_free(skb: msg);
555	return err;
556	}
557
558	static int
559	vdpa_nl_cmd_mgmtdev_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb)
560	{
561	struct vdpa_mgmt_dev *mdev;
562	int start = cb->args[0];
563	int idx = 0;
564	int err;
565
566	down_read(sem: &vdpa_dev_lock);
567	list_for_each_entry(mdev, &mdev_head, list) {
568	if (idx < start) {
569	idx++;
570	continue;
571	}
572	err = vdpa_mgmtdev_fill(mdev, msg, NETLINK_CB(cb->skb).portid,
573	seq: cb->nlh->nlmsg_seq, NLM_F_MULTI);
574	if (err)
575	goto out;
576	idx++;
577	}
578	out:
579	up_read(sem: &vdpa_dev_lock);
580	cb->args[0] = idx;
581	return msg->len;
582	}
583
584	#define VDPA_DEV_NET_ATTRS_MASK (BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) | \
585	BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU) | \
586	BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP))
587
588	/*
589	* Bitmask for all per-device features: feature bits VIRTIO_TRANSPORT_F_START
590	* through VIRTIO_TRANSPORT_F_END are unset, i.e. 0xfffffc000fffffff for
591	* all 64bit features. If the features are extended beyond 64 bits, or new
592	* "holes" are reserved for other type of features than per-device, this
593	* macro would have to be updated.
594	*/
595	#define VIRTIO_DEVICE_F_MASK (~0ULL << (VIRTIO_TRANSPORT_F_END + 1) | \
596	((1ULL << VIRTIO_TRANSPORT_F_START) - 1))
597
598	static int vdpa_nl_cmd_dev_add_set_doit(struct sk_buff *skb, struct genl_info *info)
599	{
600	struct vdpa_dev_set_config config = {};
601	struct nlattr **nl_attrs = info->attrs;
602	struct vdpa_mgmt_dev *mdev;
603	unsigned int ncls = 0;
604	const u8 *macaddr;
605	const char *name;
606	u64 classes;
607	int err = 0;
608
609	if (!info->attrs[VDPA_ATTR_DEV_NAME])
610	return -EINVAL;
611
612	name = nla_data(nla: info->attrs[VDPA_ATTR_DEV_NAME]);
613
614	if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR]) {
615	macaddr = nla_data(nla: nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR]);
616	memcpy(config.net.mac, macaddr, sizeof(config.net.mac));
617	config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR);
618	}
619	if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MTU]) {
620	config.net.mtu =
621	nla_get_u16(nla: nl_attrs[VDPA_ATTR_DEV_NET_CFG_MTU]);
622	config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU);
623	}
624	if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MAX_VQP]) {
625	config.net.max_vq_pairs =
626	nla_get_u16(nla: nl_attrs[VDPA_ATTR_DEV_NET_CFG_MAX_VQP]);
627	if (!config.net.max_vq_pairs) {
628	NL_SET_ERR_MSG_MOD(info->extack,
629	"At least one pair of VQs is required");
630	return -EINVAL;
631	}
632	config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);
633	}
634	if (nl_attrs[VDPA_ATTR_DEV_FEATURES]) {
635	u64 missing = 0x0ULL;
636
637	config.device_features =
638	nla_get_u64(nla: nl_attrs[VDPA_ATTR_DEV_FEATURES]);
639	if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR] &&
640	!(config.device_features & BIT_ULL(VIRTIO_NET_F_MAC)))
641	missing |= BIT_ULL(VIRTIO_NET_F_MAC);
642	if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MTU] &&
643	!(config.device_features & BIT_ULL(VIRTIO_NET_F_MTU)))
644	missing |= BIT_ULL(VIRTIO_NET_F_MTU);
645	if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MAX_VQP] &&
646	config.net.max_vq_pairs > 1 &&
647	!(config.device_features & BIT_ULL(VIRTIO_NET_F_MQ)))
648	missing |= BIT_ULL(VIRTIO_NET_F_MQ);
649	if (missing) {
650	NL_SET_ERR_MSG_FMT_MOD(info->extack,
651	"Missing features 0x%llx for provided attributes",
652	missing);
653	return -EINVAL;
654	}
655	config.mask |= BIT_ULL(VDPA_ATTR_DEV_FEATURES);
656	}
657
658	/* Skip checking capability if user didn't prefer to configure any
659	* device networking attributes. It is likely that user might have used
660	* a device specific method to configure such attributes or using device
661	* default attributes.
662	*/
663	if ((config.mask & VDPA_DEV_NET_ATTRS_MASK) &&
664	!netlink_capable(skb, CAP_NET_ADMIN))
665	return -EPERM;
666
667	down_write(sem: &vdpa_dev_lock);
668	mdev = vdpa_mgmtdev_get_from_attr(attrs: info->attrs);
669	if (IS_ERR(ptr: mdev)) {
670	NL_SET_ERR_MSG_MOD(info->extack, "Fail to find the specified management device");
671	err = PTR_ERR(ptr: mdev);
672	goto err;
673	}
674
675	if ((config.mask & mdev->config_attr_mask) != config.mask) {
676	NL_SET_ERR_MSG_FMT_MOD(info->extack,
677	"Some provided attributes are not supported: 0x%llx",
678	config.mask & ~mdev->config_attr_mask);
679	err = -EOPNOTSUPP;
680	goto err;
681	}
682
683	classes = vdpa_mgmtdev_get_classes(mdev, nclasses: &ncls);
684	if (config.mask & VDPA_DEV_NET_ATTRS_MASK &&
685	!(classes & BIT_ULL(VIRTIO_ID_NET))) {
686	NL_SET_ERR_MSG_MOD(info->extack,
687	"Network class attributes provided on unsupported management device");
688	err = -EINVAL;
689	goto err;
690	}
691	if (!(config.mask & VDPA_DEV_NET_ATTRS_MASK) &&
692	config.mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES) &&
693	classes & BIT_ULL(VIRTIO_ID_NET) && ncls > 1 &&
694	config.device_features & VIRTIO_DEVICE_F_MASK) {
695	NL_SET_ERR_MSG_MOD(info->extack,
696	"Management device supports multi-class while device features specified are ambiguous");
697	err = -EINVAL;
698	goto err;
699	}
700
701	err = mdev->ops->dev_add(mdev, name, &config);
702	err:
703	up_write(sem: &vdpa_dev_lock);
704	return err;
705	}
706
707	static int vdpa_nl_cmd_dev_del_set_doit(struct sk_buff *skb, struct genl_info *info)
708	{
709	struct vdpa_mgmt_dev *mdev;
710	struct vdpa_device *vdev;
711	struct device *dev;
712	const char *name;
713	int err = 0;
714
715	if (!info->attrs[VDPA_ATTR_DEV_NAME])
716	return -EINVAL;
717	name = nla_data(nla: info->attrs[VDPA_ATTR_DEV_NAME]);
718
719	down_write(sem: &vdpa_dev_lock);
720	dev = bus_find_device(bus: &vdpa_bus, NULL, data: name, match: vdpa_name_match);
721	if (!dev) {
722	NL_SET_ERR_MSG_MOD(info->extack, "device not found");
723	err = -ENODEV;
724	goto dev_err;
725	}
726	vdev = container_of(dev, struct vdpa_device, dev);
727	if (!vdev->mdev) {
728	NL_SET_ERR_MSG_MOD(info->extack, "Only user created device can be deleted by user");
729	err = -EINVAL;
730	goto mdev_err;
731	}
732	mdev = vdev->mdev;
733	mdev->ops->dev_del(mdev, vdev);
734	mdev_err:
735	put_device(dev);
736	dev_err:
737	up_write(sem: &vdpa_dev_lock);
738	return err;
739	}
740
741	static int
742	vdpa_dev_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq,
743	int flags, struct netlink_ext_ack *extack)
744	{
745	u16 max_vq_size;
746	u16 min_vq_size = 1;
747	u32 device_id;
748	u32 vendor_id;
749	void *hdr;
750	int err;
751
752	hdr = genlmsg_put(skb: msg, portid, seq, family: &vdpa_nl_family, flags, cmd: VDPA_CMD_DEV_NEW);
753	if (!hdr)
754	return -EMSGSIZE;
755
756	err = vdpa_nl_mgmtdev_handle_fill(msg, mdev: vdev->mdev);
757	if (err)
758	goto msg_err;
759
760	device_id = vdev->config->get_device_id(vdev);
761	vendor_id = vdev->config->get_vendor_id(vdev);
762	max_vq_size = vdev->config->get_vq_num_max(vdev);
763	if (vdev->config->get_vq_num_min)
764	min_vq_size = vdev->config->get_vq_num_min(vdev);
765
766	err = -EMSGSIZE;
767	if (nla_put_string(skb: msg, attrtype: VDPA_ATTR_DEV_NAME, str: dev_name(dev: &vdev->dev)))
768	goto msg_err;
769	if (nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_ID, value: device_id))
770	goto msg_err;
771	if (nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_VENDOR_ID, value: vendor_id))
772	goto msg_err;
773	if (nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_MAX_VQS, value: vdev->nvqs))
774	goto msg_err;
775	if (nla_put_u16(skb: msg, attrtype: VDPA_ATTR_DEV_MAX_VQ_SIZE, value: max_vq_size))
776	goto msg_err;
777	if (nla_put_u16(skb: msg, attrtype: VDPA_ATTR_DEV_MIN_VQ_SIZE, value: min_vq_size))
778	goto msg_err;
779
780	genlmsg_end(skb: msg, hdr);
781	return 0;
782
783	msg_err:
784	genlmsg_cancel(skb: msg, hdr);
785	return err;
786	}
787
788	static int vdpa_nl_cmd_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
789	{
790	struct vdpa_device *vdev;
791	struct sk_buff *msg;
792	const char *devname;
793	struct device *dev;
794	int err;
795
796	if (!info->attrs[VDPA_ATTR_DEV_NAME])
797	return -EINVAL;
798	devname = nla_data(nla: info->attrs[VDPA_ATTR_DEV_NAME]);
799	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
800	if (!msg)
801	return -ENOMEM;
802
803	down_read(sem: &vdpa_dev_lock);
804	dev = bus_find_device(bus: &vdpa_bus, NULL, data: devname, match: vdpa_name_match);
805	if (!dev) {
806	NL_SET_ERR_MSG_MOD(info->extack, "device not found");
807	err = -ENODEV;
808	goto err;
809	}
810	vdev = container_of(dev, struct vdpa_device, dev);
811	if (!vdev->mdev) {
812	err = -EINVAL;
813	goto mdev_err;
814	}
815	err = vdpa_dev_fill(vdev, msg, portid: info->snd_portid, seq: info->snd_seq, flags: 0, extack: info->extack);
816	if (err)
817	goto mdev_err;
818
819	err = genlmsg_reply(skb: msg, info);
820	put_device(dev);
821	up_read(sem: &vdpa_dev_lock);
822	return err;
823
824	mdev_err:
825	put_device(dev);
826	err:
827	up_read(sem: &vdpa_dev_lock);
828	nlmsg_free(skb: msg);
829	return err;
830	}
831
832	struct vdpa_dev_dump_info {
833	struct sk_buff *msg;
834	struct netlink_callback *cb;
835	int start_idx;
836	int idx;
837	};
838
839	static int vdpa_dev_dump(struct device *dev, void *data)
840	{
841	struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev);
842	struct vdpa_dev_dump_info *info = data;
843	int err;
844
845	if (!vdev->mdev)
846	return 0;
847	if (info->idx < info->start_idx) {
848	info->idx++;
849	return 0;
850	}
851	err = vdpa_dev_fill(vdev, msg: info->msg, NETLINK_CB(info->cb->skb).portid,
852	seq: info->cb->nlh->nlmsg_seq, NLM_F_MULTI, extack: info->cb->extack);
853	if (err)
854	return err;
855
856	info->idx++;
857	return 0;
858	}
859
860	static int vdpa_nl_cmd_dev_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb)
861	{
862	struct vdpa_dev_dump_info info;
863
864	info.msg = msg;
865	info.cb = cb;
866	info.start_idx = cb->args[0];
867	info.idx = 0;
868
869	down_read(sem: &vdpa_dev_lock);
870	bus_for_each_dev(bus: &vdpa_bus, NULL, data: &info, fn: vdpa_dev_dump);
871	up_read(sem: &vdpa_dev_lock);
872	cb->args[0] = info.idx;
873	return msg->len;
874	}
875
876	static int vdpa_dev_net_mq_config_fill(struct sk_buff *msg, u64 features,
877	const struct virtio_net_config *config)
878	{
879	u16 val_u16;
880
881	if ((features & BIT_ULL(VIRTIO_NET_F_MQ)) == 0 &&
882	(features & BIT_ULL(VIRTIO_NET_F_RSS)) == 0)
883	return 0;
884
885	val_u16 = __virtio16_to_cpu(little_endian: true, val: config->max_virtqueue_pairs);
886
887	return nla_put_u16(skb: msg, attrtype: VDPA_ATTR_DEV_NET_CFG_MAX_VQP, value: val_u16);
888	}
889
890	static int vdpa_dev_net_mtu_config_fill(struct sk_buff *msg, u64 features,
891	const struct virtio_net_config *config)
892	{
893	u16 val_u16;
894
895	if ((features & BIT_ULL(VIRTIO_NET_F_MTU)) == 0)
896	return 0;
897
898	val_u16 = __virtio16_to_cpu(little_endian: true, val: config->mtu);
899
900	return nla_put_u16(skb: msg, attrtype: VDPA_ATTR_DEV_NET_CFG_MTU, value: val_u16);
901	}
902
903	static int vdpa_dev_net_mac_config_fill(struct sk_buff *msg, u64 features,
904	const struct virtio_net_config *config)
905	{
906	if ((features & BIT_ULL(VIRTIO_NET_F_MAC)) == 0)
907	return 0;
908
909	return nla_put(skb: msg, attrtype: VDPA_ATTR_DEV_NET_CFG_MACADDR,
910	attrlen: sizeof(config->mac), data: config->mac);
911	}
912
913	static int vdpa_dev_net_status_config_fill(struct sk_buff *msg, u64 features,
914	const struct virtio_net_config *config)
915	{
916	u16 val_u16;
917
918	if ((features & BIT_ULL(VIRTIO_NET_F_STATUS)) == 0)
919	return 0;
920
921	val_u16 = __virtio16_to_cpu(little_endian: true, val: config->status);
922	return nla_put_u16(skb: msg, attrtype: VDPA_ATTR_DEV_NET_STATUS, value: val_u16);
923	}
924
925	static int vdpa_dev_net_config_fill(struct vdpa_device *vdev, struct sk_buff *msg)
926	{
927	struct virtio_net_config config = {};
928	u64 features_device;
929
930	vdev->config->get_config(vdev, 0, &config, sizeof(config));
931
932	features_device = vdev->config->get_device_features(vdev);
933
934	if (nla_put_u64_64bit(skb: msg, attrtype: VDPA_ATTR_DEV_FEATURES, value: features_device,
935	padattr: VDPA_ATTR_PAD))
936	return -EMSGSIZE;
937
938	if (vdpa_dev_net_mtu_config_fill(msg, features: features_device, config: &config))
939	return -EMSGSIZE;
940
941	if (vdpa_dev_net_mac_config_fill(msg, features: features_device, config: &config))
942	return -EMSGSIZE;
943
944	if (vdpa_dev_net_status_config_fill(msg, features: features_device, config: &config))
945	return -EMSGSIZE;
946
947	return vdpa_dev_net_mq_config_fill(msg, features: features_device, config: &config);
948	}
949
950	static int
951	vdpa_dev_blk_capacity_config_fill(struct sk_buff *msg,
952	const struct virtio_blk_config *config)
953	{
954	u64 val_u64;
955
956	val_u64 = __virtio64_to_cpu(little_endian: true, val: config->capacity);
957
958	return nla_put_u64_64bit(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_CAPACITY,
959	value: val_u64, padattr: VDPA_ATTR_PAD);
960	}
961
962	static int
963	vdpa_dev_blk_seg_size_config_fill(struct sk_buff *msg, u64 features,
964	const struct virtio_blk_config *config)
965	{
966	u32 val_u32;
967
968	if ((features & BIT_ULL(VIRTIO_BLK_F_SIZE_MAX)) == 0)
969	return 0;
970
971	val_u32 = __virtio32_to_cpu(little_endian: true, val: config->size_max);
972
973	return nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_SIZE_MAX, value: val_u32);
974	}
975
976	/* fill the block size*/
977	static int
978	vdpa_dev_blk_block_size_config_fill(struct sk_buff *msg, u64 features,
979	const struct virtio_blk_config *config)
980	{
981	u32 val_u32;
982
983	if ((features & BIT_ULL(VIRTIO_BLK_F_BLK_SIZE)) == 0)
984	return 0;
985
986	val_u32 = __virtio32_to_cpu(little_endian: true, val: config->blk_size);
987
988	return nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_BLK_SIZE, value: val_u32);
989	}
990
991	static int
992	vdpa_dev_blk_seg_max_config_fill(struct sk_buff *msg, u64 features,
993	const struct virtio_blk_config *config)
994	{
995	u32 val_u32;
996
997	if ((features & BIT_ULL(VIRTIO_BLK_F_SEG_MAX)) == 0)
998	return 0;
999
1000	val_u32 = __virtio32_to_cpu(little_endian: true, val: config->seg_max);
1001
1002	return nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_SEG_MAX, value: val_u32);
1003	}
1004
1005	static int vdpa_dev_blk_mq_config_fill(struct sk_buff *msg, u64 features,
1006	const struct virtio_blk_config *config)
1007	{
1008	u16 val_u16;
1009
1010	if ((features & BIT_ULL(VIRTIO_BLK_F_MQ)) == 0)
1011	return 0;
1012
1013	val_u16 = __virtio16_to_cpu(little_endian: true, val: config->num_queues);
1014
1015	return nla_put_u16(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_NUM_QUEUES, value: val_u16);
1016	}
1017
1018	static int vdpa_dev_blk_topology_config_fill(struct sk_buff *msg, u64 features,
1019	const struct virtio_blk_config *config)
1020	{
1021	u16 min_io_size;
1022	u32 opt_io_size;
1023
1024	if ((features & BIT_ULL(VIRTIO_BLK_F_TOPOLOGY)) == 0)
1025	return 0;
1026
1027	min_io_size = __virtio16_to_cpu(little_endian: true, val: config->min_io_size);
1028	opt_io_size = __virtio32_to_cpu(little_endian: true, val: config->opt_io_size);
1029
1030	if (nla_put_u8(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_PHY_BLK_EXP,
1031	value: config->physical_block_exp))
1032	return -EMSGSIZE;
1033
1034	if (nla_put_u8(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_ALIGN_OFFSET,
1035	value: config->alignment_offset))
1036	return -EMSGSIZE;
1037
1038	if (nla_put_u16(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_MIN_IO_SIZE, value: min_io_size))
1039	return -EMSGSIZE;
1040
1041	if (nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_OPT_IO_SIZE, value: opt_io_size))
1042	return -EMSGSIZE;
1043
1044	return 0;
1045	}
1046
1047	static int vdpa_dev_blk_discard_config_fill(struct sk_buff *msg, u64 features,
1048	const struct virtio_blk_config *config)
1049	{
1050	u32 val_u32;
1051
1052	if ((features & BIT_ULL(VIRTIO_BLK_F_DISCARD)) == 0)
1053	return 0;
1054
1055	val_u32 = __virtio32_to_cpu(little_endian: true, val: config->max_discard_sectors);
1056	if (nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_MAX_DISCARD_SEC, value: val_u32))
1057	return -EMSGSIZE;
1058
1059	val_u32 = __virtio32_to_cpu(little_endian: true, val: config->max_discard_seg);
1060	if (nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_MAX_DISCARD_SEG, value: val_u32))
1061	return -EMSGSIZE;
1062
1063	val_u32 = __virtio32_to_cpu(little_endian: true, val: config->discard_sector_alignment);
1064	if (nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_DISCARD_SEC_ALIGN, value: val_u32))
1065	return -EMSGSIZE;
1066
1067	return 0;
1068	}
1069
1070	static int
1071	vdpa_dev_blk_write_zeroes_config_fill(struct sk_buff *msg, u64 features,
1072	const struct virtio_blk_config *config)
1073	{
1074	u32 val_u32;
1075
1076	if ((features & BIT_ULL(VIRTIO_BLK_F_WRITE_ZEROES)) == 0)
1077	return 0;
1078
1079	val_u32 = __virtio32_to_cpu(little_endian: true, val: config->max_write_zeroes_sectors);
1080	if (nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEC, value: val_u32))
1081	return -EMSGSIZE;
1082
1083	val_u32 = __virtio32_to_cpu(little_endian: true, val: config->max_write_zeroes_seg);
1084	if (nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_CFG_MAX_WRITE_ZEROES_SEG, value: val_u32))
1085	return -EMSGSIZE;
1086
1087	return 0;
1088	}
1089
1090	static int vdpa_dev_blk_ro_config_fill(struct sk_buff *msg, u64 features)
1091	{
1092	u8 ro;
1093
1094	ro = ((features & BIT_ULL(VIRTIO_BLK_F_RO)) == 0) ? 0 : 1;
1095	if (nla_put_u8(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_READ_ONLY, value: ro))
1096	return -EMSGSIZE;
1097
1098	return 0;
1099	}
1100
1101	static int vdpa_dev_blk_flush_config_fill(struct sk_buff *msg, u64 features)
1102	{
1103	u8 flush;
1104
1105	flush = ((features & BIT_ULL(VIRTIO_BLK_F_FLUSH)) == 0) ? 0 : 1;
1106	if (nla_put_u8(skb: msg, attrtype: VDPA_ATTR_DEV_BLK_FLUSH, value: flush))
1107	return -EMSGSIZE;
1108
1109	return 0;
1110	}
1111
1112	static int vdpa_dev_blk_config_fill(struct vdpa_device *vdev,
1113	struct sk_buff *msg)
1114	{
1115	struct virtio_blk_config config = {};
1116	u64 features_device;
1117
1118	vdev->config->get_config(vdev, 0, &config, sizeof(config));
1119
1120	features_device = vdev->config->get_device_features(vdev);
1121
1122	if (nla_put_u64_64bit(skb: msg, attrtype: VDPA_ATTR_DEV_FEATURES, value: features_device,
1123	padattr: VDPA_ATTR_PAD))
1124	return -EMSGSIZE;
1125
1126	if (vdpa_dev_blk_capacity_config_fill(msg, config: &config))
1127	return -EMSGSIZE;
1128
1129	if (vdpa_dev_blk_seg_size_config_fill(msg, features: features_device, config: &config))
1130	return -EMSGSIZE;
1131
1132	if (vdpa_dev_blk_block_size_config_fill(msg, features: features_device, config: &config))
1133	return -EMSGSIZE;
1134
1135	if (vdpa_dev_blk_seg_max_config_fill(msg, features: features_device, config: &config))
1136	return -EMSGSIZE;
1137
1138	if (vdpa_dev_blk_mq_config_fill(msg, features: features_device, config: &config))
1139	return -EMSGSIZE;
1140
1141	if (vdpa_dev_blk_topology_config_fill(msg, features: features_device, config: &config))
1142	return -EMSGSIZE;
1143
1144	if (vdpa_dev_blk_discard_config_fill(msg, features: features_device, config: &config))
1145	return -EMSGSIZE;
1146
1147	if (vdpa_dev_blk_write_zeroes_config_fill(msg, features: features_device, config: &config))
1148	return -EMSGSIZE;
1149
1150	if (vdpa_dev_blk_ro_config_fill(msg, features: features_device))
1151	return -EMSGSIZE;
1152
1153	if (vdpa_dev_blk_flush_config_fill(msg, features: features_device))
1154	return -EMSGSIZE;
1155
1156	return 0;
1157	}
1158
1159	static int
1160	vdpa_dev_config_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq,
1161	int flags, struct netlink_ext_ack *extack)
1162	{
1163	u64 features_driver;
1164	u8 status = 0;
1165	u32 device_id;
1166	void *hdr;
1167	int err;
1168
1169	down_read(sem: &vdev->cf_lock);
1170	hdr = genlmsg_put(skb: msg, portid, seq, family: &vdpa_nl_family, flags,
1171	cmd: VDPA_CMD_DEV_CONFIG_GET);
1172	if (!hdr) {
1173	err = -EMSGSIZE;
1174	goto out;
1175	}
1176
1177	if (nla_put_string(skb: msg, attrtype: VDPA_ATTR_DEV_NAME, str: dev_name(dev: &vdev->dev))) {
1178	err = -EMSGSIZE;
1179	goto msg_err;
1180	}
1181
1182	device_id = vdev->config->get_device_id(vdev);
1183	if (nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_ID, value: device_id)) {
1184	err = -EMSGSIZE;
1185	goto msg_err;
1186	}
1187
1188	/* only read driver features after the feature negotiation is done */
1189	status = vdev->config->get_status(vdev);
1190	if (status & VIRTIO_CONFIG_S_FEATURES_OK) {
1191	features_driver = vdev->config->get_driver_features(vdev);
1192	if (nla_put_u64_64bit(skb: msg, attrtype: VDPA_ATTR_DEV_NEGOTIATED_FEATURES, value: features_driver,
1193	padattr: VDPA_ATTR_PAD)) {
1194	err = -EMSGSIZE;
1195	goto msg_err;
1196	}
1197	}
1198
1199	switch (device_id) {
1200	case VIRTIO_ID_NET:
1201	err = vdpa_dev_net_config_fill(vdev, msg);
1202	break;
1203	case VIRTIO_ID_BLOCK:
1204	err = vdpa_dev_blk_config_fill(vdev, msg);
1205	break;
1206	default:
1207	err = -EOPNOTSUPP;
1208	break;
1209	}
1210	if (err)
1211	goto msg_err;
1212
1213	up_read(sem: &vdev->cf_lock);
1214	genlmsg_end(skb: msg, hdr);
1215	return 0;
1216
1217	msg_err:
1218	genlmsg_cancel(skb: msg, hdr);
1219	out:
1220	up_read(sem: &vdev->cf_lock);
1221	return err;
1222	}
1223
1224	static int vdpa_fill_stats_rec(struct vdpa_device *vdev, struct sk_buff *msg,
1225	struct genl_info *info, u32 index)
1226	{
1227	struct virtio_net_config config = {};
1228	u64 features;
1229	u8 status;
1230	int err;
1231
1232	status = vdev->config->get_status(vdev);
1233	if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) {
1234	NL_SET_ERR_MSG_MOD(info->extack, "feature negotiation not complete");
1235	return -EAGAIN;
1236	}
1237	vdpa_get_config_unlocked(vdev, offset: 0, buf: &config, len: sizeof(config));
1238
1239	features = vdev->config->get_driver_features(vdev);
1240	if (nla_put_u64_64bit(skb: msg, attrtype: VDPA_ATTR_DEV_NEGOTIATED_FEATURES,
1241	value: features, padattr: VDPA_ATTR_PAD))
1242	return -EMSGSIZE;
1243
1244	err = vdpa_dev_net_mq_config_fill(msg, features, config: &config);
1245	if (err)
1246	return err;
1247
1248	if (nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_QUEUE_INDEX, value: index))
1249	return -EMSGSIZE;
1250
1251	err = vdev->config->get_vendor_vq_stats(vdev, index, msg, info->extack);
1252	if (err)
1253	return err;
1254
1255	return 0;
1256	}
1257
1258	static int vendor_stats_fill(struct vdpa_device *vdev, struct sk_buff *msg,
1259	struct genl_info *info, u32 index)
1260	{
1261	int err;
1262
1263	down_read(sem: &vdev->cf_lock);
1264	if (!vdev->config->get_vendor_vq_stats) {
1265	err = -EOPNOTSUPP;
1266	goto out;
1267	}
1268
1269	err = vdpa_fill_stats_rec(vdev, msg, info, index);
1270	out:
1271	up_read(sem: &vdev->cf_lock);
1272	return err;
1273	}
1274
1275	static int vdpa_dev_vendor_stats_fill(struct vdpa_device *vdev,
1276	struct sk_buff *msg,
1277	struct genl_info *info, u32 index)
1278	{
1279	u32 device_id;
1280	void *hdr;
1281	int err;
1282	u32 portid = info->snd_portid;
1283	u32 seq = info->snd_seq;
1284	u32 flags = 0;
1285
1286	hdr = genlmsg_put(skb: msg, portid, seq, family: &vdpa_nl_family, flags,
1287	cmd: VDPA_CMD_DEV_VSTATS_GET);
1288	if (!hdr)
1289	return -EMSGSIZE;
1290
1291	if (nla_put_string(skb: msg, attrtype: VDPA_ATTR_DEV_NAME, str: dev_name(dev: &vdev->dev))) {
1292	err = -EMSGSIZE;
1293	goto undo_msg;
1294	}
1295
1296	device_id = vdev->config->get_device_id(vdev);
1297	if (nla_put_u32(skb: msg, attrtype: VDPA_ATTR_DEV_ID, value: device_id)) {
1298	err = -EMSGSIZE;
1299	goto undo_msg;
1300	}
1301
1302	switch (device_id) {
1303	case VIRTIO_ID_NET:
1304	if (index > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) {
1305	NL_SET_ERR_MSG_MOD(info->extack, "queue index exceeds max value");
1306	err = -ERANGE;
1307	break;
1308	}
1309
1310	err = vendor_stats_fill(vdev, msg, info, index);
1311	break;
1312	default:
1313	err = -EOPNOTSUPP;
1314	break;
1315	}
1316	genlmsg_end(skb: msg, hdr);
1317
1318	return err;
1319
1320	undo_msg:
1321	genlmsg_cancel(skb: msg, hdr);
1322	return err;
1323	}
1324
1325	static int vdpa_nl_cmd_dev_config_get_doit(struct sk_buff *skb, struct genl_info *info)
1326	{
1327	struct vdpa_device *vdev;
1328	struct sk_buff *msg;
1329	const char *devname;
1330	struct device *dev;
1331	int err;
1332
1333	if (!info->attrs[VDPA_ATTR_DEV_NAME])
1334	return -EINVAL;
1335	devname = nla_data(nla: info->attrs[VDPA_ATTR_DEV_NAME]);
1336	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1337	if (!msg)
1338	return -ENOMEM;
1339
1340	down_read(sem: &vdpa_dev_lock);
1341	dev = bus_find_device(bus: &vdpa_bus, NULL, data: devname, match: vdpa_name_match);
1342	if (!dev) {
1343	NL_SET_ERR_MSG_MOD(info->extack, "device not found");
1344	err = -ENODEV;
1345	goto dev_err;
1346	}
1347	vdev = container_of(dev, struct vdpa_device, dev);
1348	if (!vdev->mdev) {
1349	NL_SET_ERR_MSG_MOD(info->extack, "unmanaged vdpa device");
1350	err = -EINVAL;
1351	goto mdev_err;
1352	}
1353	err = vdpa_dev_config_fill(vdev, msg, portid: info->snd_portid, seq: info->snd_seq,
1354	flags: 0, extack: info->extack);
1355	if (!err)
1356	err = genlmsg_reply(skb: msg, info);
1357
1358	mdev_err:
1359	put_device(dev);
1360	dev_err:
1361	up_read(sem: &vdpa_dev_lock);
1362	if (err)
1363	nlmsg_free(skb: msg);
1364	return err;
1365	}
1366
1367	static int vdpa_dev_net_device_attr_set(struct vdpa_device *vdev,
1368	struct genl_info *info)
1369	{
1370	struct vdpa_dev_set_config set_config = {};
1371	struct vdpa_mgmt_dev *mdev = vdev->mdev;
1372	struct nlattr **nl_attrs = info->attrs;
1373	const u8 *macaddr;
1374	int err = -EOPNOTSUPP;
1375
1376	down_write(sem: &vdev->cf_lock);
1377	if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR]) {
1378	set_config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR);
1379	macaddr = nla_data(nla: nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR]);
1380
1381	if (is_valid_ether_addr(addr: macaddr)) {
1382	ether_addr_copy(dst: set_config.net.mac, src: macaddr);
1383	if (mdev->ops->dev_set_attr) {
1384	err = mdev->ops->dev_set_attr(mdev, vdev,
1385	&set_config);
1386	} else {
1387	NL_SET_ERR_MSG_FMT_MOD(info->extack,
1388	"Operation not supported by the device.");
1389	}
1390	} else {
1391	NL_SET_ERR_MSG_FMT_MOD(info->extack,
1392	"Invalid MAC address");
1393	}
1394	}
1395	up_write(sem: &vdev->cf_lock);
1396	return err;
1397	}
1398
1399	static int vdpa_nl_cmd_dev_attr_set_doit(struct sk_buff *skb,
1400	struct genl_info *info)
1401	{
1402	struct vdpa_device *vdev;
1403	struct device *dev;
1404	const char *name;
1405	u64 classes;
1406	int err = 0;
1407
1408	if (!info->attrs[VDPA_ATTR_DEV_NAME])
1409	return -EINVAL;
1410
1411	name = nla_data(nla: info->attrs[VDPA_ATTR_DEV_NAME]);
1412
1413	down_write(sem: &vdpa_dev_lock);
1414	dev = bus_find_device(bus: &vdpa_bus, NULL, data: name, match: vdpa_name_match);
1415	if (!dev) {
1416	NL_SET_ERR_MSG_MOD(info->extack, "device not found");
1417	err = -ENODEV;
1418	goto dev_err;
1419	}
1420	vdev = container_of(dev, struct vdpa_device, dev);
1421	if (!vdev->mdev) {
1422	NL_SET_ERR_MSG_MOD(info->extack, "unmanaged vdpa device");
1423	err = -EINVAL;
1424	goto mdev_err;
1425	}
1426	classes = vdpa_mgmtdev_get_classes(mdev: vdev->mdev, NULL);
1427	if (classes & BIT_ULL(VIRTIO_ID_NET)) {
1428	err = vdpa_dev_net_device_attr_set(vdev, info);
1429	} else {
1430	NL_SET_ERR_MSG_FMT_MOD(info->extack, "%s device not supported",
1431	name);
1432	}
1433
1434	mdev_err:
1435	put_device(dev);
1436	dev_err:
1437	up_write(sem: &vdpa_dev_lock);
1438	return err;
1439	}
1440
1441	static int vdpa_dev_config_dump(struct device *dev, void *data)
1442	{
1443	struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev);
1444	struct vdpa_dev_dump_info *info = data;
1445	int err;
1446
1447	if (!vdev->mdev)
1448	return 0;
1449	if (info->idx < info->start_idx) {
1450	info->idx++;
1451	return 0;
1452	}
1453	err = vdpa_dev_config_fill(vdev, msg: info->msg, NETLINK_CB(info->cb->skb).portid,
1454	seq: info->cb->nlh->nlmsg_seq, NLM_F_MULTI,
1455	extack: info->cb->extack);
1456	if (err)
1457	return err;
1458
1459	info->idx++;
1460	return 0;
1461	}
1462
1463	static int
1464	vdpa_nl_cmd_dev_config_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb)
1465	{
1466	struct vdpa_dev_dump_info info;
1467
1468	info.msg = msg;
1469	info.cb = cb;
1470	info.start_idx = cb->args[0];
1471	info.idx = 0;
1472
1473	down_read(sem: &vdpa_dev_lock);
1474	bus_for_each_dev(bus: &vdpa_bus, NULL, data: &info, fn: vdpa_dev_config_dump);
1475	up_read(sem: &vdpa_dev_lock);
1476	cb->args[0] = info.idx;
1477	return msg->len;
1478	}
1479
1480	static int vdpa_nl_cmd_dev_stats_get_doit(struct sk_buff *skb,
1481	struct genl_info *info)
1482	{
1483	struct vdpa_device *vdev;
1484	struct sk_buff *msg;
1485	const char *devname;
1486	struct device *dev;
1487	u32 index;
1488	int err;
1489
1490	if (!info->attrs[VDPA_ATTR_DEV_NAME])
1491	return -EINVAL;
1492
1493	if (!info->attrs[VDPA_ATTR_DEV_QUEUE_INDEX])
1494	return -EINVAL;
1495
1496	devname = nla_data(nla: info->attrs[VDPA_ATTR_DEV_NAME]);
1497	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1498	if (!msg)
1499	return -ENOMEM;
1500
1501	index = nla_get_u32(nla: info->attrs[VDPA_ATTR_DEV_QUEUE_INDEX]);
1502	down_read(sem: &vdpa_dev_lock);
1503	dev = bus_find_device(bus: &vdpa_bus, NULL, data: devname, match: vdpa_name_match);
1504	if (!dev) {
1505	NL_SET_ERR_MSG_MOD(info->extack, "device not found");
1506	err = -ENODEV;
1507	goto dev_err;
1508	}
1509	vdev = container_of(dev, struct vdpa_device, dev);
1510	if (!vdev->mdev) {
1511	NL_SET_ERR_MSG_MOD(info->extack, "unmanaged vdpa device");
1512	err = -EINVAL;
1513	goto mdev_err;
1514	}
1515	err = vdpa_dev_vendor_stats_fill(vdev, msg, info, index);
1516	if (err)
1517	goto mdev_err;
1518
1519	err = genlmsg_reply(skb: msg, info);
1520
1521	put_device(dev);
1522	up_read(sem: &vdpa_dev_lock);
1523
1524	return err;
1525
1526	mdev_err:
1527	put_device(dev);
1528	dev_err:
1529	nlmsg_free(skb: msg);
1530	up_read(sem: &vdpa_dev_lock);
1531	return err;
1532	}
1533
1534	static const struct nla_policy vdpa_nl_policy[VDPA_ATTR_MAX + 1] = {
1535	[VDPA_ATTR_MGMTDEV_BUS_NAME] = { .type = NLA_NUL_STRING },
1536	[VDPA_ATTR_MGMTDEV_DEV_NAME] = { .type = NLA_STRING },
1537	[VDPA_ATTR_DEV_NAME] = { .type = NLA_STRING },
1538	[VDPA_ATTR_DEV_NET_CFG_MACADDR] = NLA_POLICY_ETH_ADDR,
1539	[VDPA_ATTR_DEV_NET_CFG_MAX_VQP] = { .type = NLA_U16 },
1540	/* virtio spec 1.1 section 5.1.4.1 for valid MTU range */
1541	[VDPA_ATTR_DEV_NET_CFG_MTU] = NLA_POLICY_MIN(NLA_U16, 68),
1542	[VDPA_ATTR_DEV_QUEUE_INDEX] = { .type = NLA_U32 },
1543	[VDPA_ATTR_DEV_FEATURES] = { .type = NLA_U64 },
1544	};
1545
1546	static const struct genl_ops vdpa_nl_ops[] = {
1547	{
1548	.cmd = VDPA_CMD_MGMTDEV_GET,
1549	.doit = vdpa_nl_cmd_mgmtdev_get_doit,
1550	.dumpit = vdpa_nl_cmd_mgmtdev_get_dumpit,
1551	},
1552	{
1553	.cmd = VDPA_CMD_DEV_NEW,
1554	.doit = vdpa_nl_cmd_dev_add_set_doit,
1555	.flags = GENL_ADMIN_PERM,
1556	},
1557	{
1558	.cmd = VDPA_CMD_DEV_DEL,
1559	.doit = vdpa_nl_cmd_dev_del_set_doit,
1560	.flags = GENL_ADMIN_PERM,
1561	},
1562	{
1563	.cmd = VDPA_CMD_DEV_GET,
1564	.doit = vdpa_nl_cmd_dev_get_doit,
1565	.dumpit = vdpa_nl_cmd_dev_get_dumpit,
1566	},
1567	{
1568	.cmd = VDPA_CMD_DEV_CONFIG_GET,
1569	.doit = vdpa_nl_cmd_dev_config_get_doit,
1570	.dumpit = vdpa_nl_cmd_dev_config_get_dumpit,
1571	},
1572	{
1573	.cmd = VDPA_CMD_DEV_VSTATS_GET,
1574	.doit = vdpa_nl_cmd_dev_stats_get_doit,
1575	.flags = GENL_ADMIN_PERM,
1576	},
1577	{
1578	.cmd = VDPA_CMD_DEV_ATTR_SET,
1579	.doit = vdpa_nl_cmd_dev_attr_set_doit,
1580	.flags = GENL_ADMIN_PERM,
1581	},
1582	};
1583
1584	static struct genl_family vdpa_nl_family __ro_after_init = {
1585	.name = VDPA_GENL_NAME,
1586	.version = VDPA_GENL_VERSION,
1587	.maxattr = VDPA_ATTR_MAX,
1588	.policy = vdpa_nl_policy,
1589	.netnsok = false,
1590	.module = THIS_MODULE,
1591	.ops = vdpa_nl_ops,
1592	.n_ops = ARRAY_SIZE(vdpa_nl_ops),
1593	.resv_start_op = VDPA_CMD_DEV_VSTATS_GET + 1,
1594	};
1595
1596	static int vdpa_init(void)
1597	{
1598	int err;
1599
1600	err = bus_register(bus: &vdpa_bus);
1601	if (err)
1602	return err;
1603	err = genl_register_family(family: &vdpa_nl_family);
1604	if (err)
1605	goto err;
1606	return 0;
1607
1608	err:
1609	bus_unregister(bus: &vdpa_bus);
1610	return err;
1611	}
1612
1613	static void __exit vdpa_exit(void)
1614	{
1615	genl_unregister_family(family: &vdpa_nl_family);
1616	bus_unregister(bus: &vdpa_bus);
1617	ida_destroy(ida: &vdpa_index_ida);
1618	}
1619	core_initcall(vdpa_init);
1620	module_exit(vdpa_exit);
1621
1622	MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>");
1623	MODULE_DESCRIPTION("vDPA bus");
1624	MODULE_LICENSE("GPL v2");
1625