1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VIRTIO based driver for vDPA device
4 *
5 * Copyright (c) 2020, Red Hat. All rights reserved.
6 * Author: Jason Wang <jasowang@redhat.com>
7 *
8 */
9
10#include <linux/init.h>
11#include <linux/module.h>
12#include <linux/device.h>
13#include <linux/kernel.h>
14#include <linux/slab.h>
15#include <linux/uuid.h>
16#include <linux/group_cpus.h>
17#include <linux/virtio.h>
18#include <linux/vdpa.h>
19#include <linux/virtio_config.h>
20#include <linux/virtio_ring.h>
21
22#define MOD_VERSION "0.1"
23#define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>"
24#define MOD_DESC "vDPA bus driver for virtio devices"
25#define MOD_LICENSE "GPL v2"
26
27struct virtio_vdpa_device {
28 struct virtio_device vdev;
29 struct vdpa_device *vdpa;
30 u64 features;
31};
32
33static inline struct virtio_vdpa_device *
34to_virtio_vdpa_device(struct virtio_device *dev)
35{
36 return container_of(dev, struct virtio_vdpa_device, vdev);
37}
38
39static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev)
40{
41 return to_virtio_vdpa_device(dev: vdev)->vdpa;
42}
43
44static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset,
45 void *buf, unsigned int len)
46{
47 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
48
49 vdpa_get_config(vdev: vdpa, offset, buf, len);
50}
51
52static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset,
53 const void *buf, unsigned int len)
54{
55 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
56
57 vdpa_set_config(dev: vdpa, offset, buf, length: len);
58}
59
60static u32 virtio_vdpa_generation(struct virtio_device *vdev)
61{
62 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
63 const struct vdpa_config_ops *ops = vdpa->config;
64
65 if (ops->get_generation)
66 return ops->get_generation(vdpa);
67
68 return 0;
69}
70
71static u8 virtio_vdpa_get_status(struct virtio_device *vdev)
72{
73 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
74 const struct vdpa_config_ops *ops = vdpa->config;
75
76 return ops->get_status(vdpa);
77}
78
79static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
80{
81 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
82
83 vdpa_set_status(vdev: vdpa, status);
84}
85
86static void virtio_vdpa_reset(struct virtio_device *vdev)
87{
88 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
89
90 vdpa_reset(vdev: vdpa, flags: 0);
91}
92
93static bool virtio_vdpa_notify(struct virtqueue *vq)
94{
95 struct vdpa_device *vdpa = vd_get_vdpa(vdev: vq->vdev);
96 const struct vdpa_config_ops *ops = vdpa->config;
97
98 ops->kick_vq(vdpa, vq->index);
99
100 return true;
101}
102
103static bool virtio_vdpa_notify_with_data(struct virtqueue *vq)
104{
105 struct vdpa_device *vdpa = vd_get_vdpa(vdev: vq->vdev);
106 const struct vdpa_config_ops *ops = vdpa->config;
107 u32 data = vring_notification_data(vq: vq);
108
109 ops->kick_vq_with_data(vdpa, data);
110
111 return true;
112}
113
114static irqreturn_t virtio_vdpa_config_cb(void *private)
115{
116 struct virtio_vdpa_device *vd_dev = private;
117
118 virtio_config_changed(dev: &vd_dev->vdev);
119
120 return IRQ_HANDLED;
121}
122
123static irqreturn_t virtio_vdpa_virtqueue_cb(void *private)
124{
125 struct virtqueue *vq = private;
126
127 return vring_interrupt(irq: 0, vq: vq);
128}
129
130static struct virtqueue *
131virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
132 void (*callback)(struct virtqueue *vq),
133 const char *name, bool ctx)
134{
135 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
136 const struct vdpa_config_ops *ops = vdpa->config;
137 bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify;
138 struct vdpa_callback cb;
139 struct virtqueue *vq;
140 u64 desc_addr, driver_addr, device_addr;
141 union virtio_map map = {0};
142 /* Assume split virtqueue, switch to packed if necessary */
143 struct vdpa_vq_state state = {0};
144 u32 align, max_num, min_num = 1;
145 bool may_reduce_num = true;
146 int err;
147
148 if (!name)
149 return NULL;
150
151 if (index >= vdpa->nvqs)
152 return ERR_PTR(error: -ENOENT);
153
154 /* We cannot accept VIRTIO_F_NOTIFICATION_DATA without kick_vq_with_data */
155 if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) {
156 if (ops->kick_vq_with_data)
157 notify = virtio_vdpa_notify_with_data;
158 else
159 __virtio_clear_bit(vdev, VIRTIO_F_NOTIFICATION_DATA);
160 }
161
162 /* Queue shouldn't already be set up. */
163 if (ops->get_vq_ready(vdpa, index))
164 return ERR_PTR(error: -ENOENT);
165
166 if (ops->get_vq_size)
167 max_num = ops->get_vq_size(vdpa, index);
168 else
169 max_num = ops->get_vq_num_max(vdpa);
170
171 if (max_num == 0) {
172 err = -ENOENT;
173 goto error_new_virtqueue;
174 }
175
176 if (ops->get_vq_num_min)
177 min_num = ops->get_vq_num_min(vdpa);
178
179 may_reduce_num = (max_num != min_num);
180
181 /* Create the vring */
182 align = ops->get_vq_align(vdpa);
183
184 if (ops->get_vq_map)
185 map = ops->get_vq_map(vdpa, index);
186 else
187 map = vdpa_get_map(vdev: vdpa);
188
189 vq = vring_create_virtqueue_map(index, num: max_num, vring_align: align, vdev,
190 weak_barriers: true, may_reduce_num, ctx,
191 notify, callback, name, map);
192 if (!vq) {
193 err = -ENOMEM;
194 goto error_new_virtqueue;
195 }
196
197 if (index == 0)
198 vdev->vmap = map;
199
200 vq->num_max = max_num;
201
202 /* Setup virtqueue callback */
203 cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL;
204 cb.private = vq;
205 cb.trigger = NULL;
206 ops->set_vq_cb(vdpa, index, &cb);
207 ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));
208
209 desc_addr = virtqueue_get_desc_addr(vq);
210 driver_addr = virtqueue_get_avail_addr(vq);
211 device_addr = virtqueue_get_used_addr(vq);
212
213 if (ops->set_vq_address(vdpa, index,
214 desc_addr, driver_addr,
215 device_addr)) {
216 err = -EINVAL;
217 goto err_vq;
218 }
219
220 /* reset virtqueue state index */
221 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
222 struct vdpa_vq_state_packed *s = &state.packed;
223
224 s->last_avail_counter = 1;
225 s->last_avail_idx = 0;
226 s->last_used_counter = 1;
227 s->last_used_idx = 0;
228 }
229 err = ops->set_vq_state(vdpa, index, &state);
230 if (err)
231 goto err_vq;
232
233 ops->set_vq_ready(vdpa, index, 1);
234
235 return vq;
236
237err_vq:
238 vring_del_virtqueue(vq);
239error_new_virtqueue:
240 ops->set_vq_ready(vdpa, index, 0);
241 /* VDPA driver should make sure vq is stopeed here */
242 WARN_ON(ops->get_vq_ready(vdpa, index));
243 return ERR_PTR(error: err);
244}
245
246static void virtio_vdpa_del_vq(struct virtqueue *vq)
247{
248 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vq->vdev);
249 struct vdpa_device *vdpa = vd_dev->vdpa;
250 const struct vdpa_config_ops *ops = vdpa->config;
251 unsigned int index = vq->index;
252
253 /* Select and deactivate the queue (best effort) */
254 ops->set_vq_ready(vdpa, index, 0);
255
256 vring_del_virtqueue(vq);
257}
258
259static void virtio_vdpa_del_vqs(struct virtio_device *vdev)
260{
261 struct virtqueue *vq, *n;
262
263 list_for_each_entry_safe(vq, n, &vdev->vqs, list)
264 virtio_vdpa_del_vq(vq);
265}
266
267static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs)
268{
269 affd->nr_sets = 1;
270 affd->set_size[0] = affvecs;
271}
272
273static struct cpumask *
274create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd)
275{
276 unsigned int affvecs = 0, curvec, usedvecs, i;
277 struct cpumask *masks = NULL;
278
279 if (nvecs > affd->pre_vectors + affd->post_vectors)
280 affvecs = nvecs - affd->pre_vectors - affd->post_vectors;
281
282 if (!affd->calc_sets)
283 affd->calc_sets = default_calc_sets;
284
285 affd->calc_sets(affd, affvecs);
286
287 if (!affvecs)
288 return NULL;
289
290 masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);
291 if (!masks)
292 return NULL;
293
294 /* Fill out vectors at the beginning that don't need affinity */
295 for (curvec = 0; curvec < affd->pre_vectors; curvec++)
296 cpumask_setall(dstp: &masks[curvec]);
297
298 for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) {
299 unsigned int this_vecs = affd->set_size[i];
300 unsigned int nr_masks;
301 int j;
302 struct cpumask *result = group_cpus_evenly(numgrps: this_vecs, nummasks: &nr_masks);
303
304 if (!result) {
305 kfree(objp: masks);
306 return NULL;
307 }
308
309 for (j = 0; j < nr_masks; j++)
310 cpumask_copy(dstp: &masks[curvec + j], srcp: &result[j]);
311 kfree(objp: result);
312
313 curvec += nr_masks;
314 usedvecs += nr_masks;
315 }
316
317 /* Fill out vectors at the end that don't need affinity */
318 if (usedvecs >= affvecs)
319 curvec = affd->pre_vectors + affvecs;
320 else
321 curvec = affd->pre_vectors + usedvecs;
322 for (; curvec < nvecs; curvec++)
323 cpumask_setall(dstp: &masks[curvec]);
324
325 return masks;
326}
327
328static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
329 struct virtqueue *vqs[],
330 struct virtqueue_info vqs_info[],
331 struct irq_affinity *desc)
332{
333 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vdev);
334 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
335 const struct vdpa_config_ops *ops = vdpa->config;
336 struct cpumask *masks;
337 struct vdpa_callback cb;
338 bool has_affinity = desc && ops->set_vq_affinity;
339 int i, err, queue_idx = 0;
340
341 if (has_affinity) {
342 masks = create_affinity_masks(nvecs: nvqs, affd: desc);
343 if (!masks)
344 return -ENOMEM;
345 }
346
347 for (i = 0; i < nvqs; ++i) {
348 struct virtqueue_info *vqi = &vqs_info[i];
349
350 if (!vqi->name) {
351 vqs[i] = NULL;
352 continue;
353 }
354
355 vqs[i] = virtio_vdpa_setup_vq(vdev, index: queue_idx++, callback: vqi->callback,
356 name: vqi->name, ctx: vqi->ctx);
357 if (IS_ERR(ptr: vqs[i])) {
358 err = PTR_ERR(ptr: vqs[i]);
359 goto err_setup_vq;
360 }
361
362 if (has_affinity)
363 ops->set_vq_affinity(vdpa, i, &masks[i]);
364 }
365
366 cb.callback = virtio_vdpa_config_cb;
367 cb.private = vd_dev;
368 ops->set_config_cb(vdpa, &cb);
369 if (has_affinity)
370 kfree(objp: masks);
371
372 return 0;
373
374err_setup_vq:
375 virtio_vdpa_del_vqs(vdev);
376 if (has_affinity)
377 kfree(objp: masks);
378 return err;
379}
380
381static u64 virtio_vdpa_get_features(struct virtio_device *vdev)
382{
383 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
384 const struct vdpa_config_ops *ops = vdpa->config;
385
386 return ops->get_device_features(vdpa);
387}
388
389static int virtio_vdpa_finalize_features(struct virtio_device *vdev)
390{
391 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
392
393 /* Give virtio_ring a chance to accept features. */
394 vring_transport_features(vdev);
395
396 return vdpa_set_features(vdev: vdpa, features: vdev->features);
397}
398
399static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)
400{
401 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vdev);
402 struct vdpa_device *vdpa = vd_dev->vdpa;
403
404 return dev_name(dev: &vdpa->dev);
405}
406
407static int virtio_vdpa_set_vq_affinity(struct virtqueue *vq,
408 const struct cpumask *cpu_mask)
409{
410 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(dev: vq->vdev);
411 struct vdpa_device *vdpa = vd_dev->vdpa;
412 const struct vdpa_config_ops *ops = vdpa->config;
413 unsigned int index = vq->index;
414
415 if (ops->set_vq_affinity)
416 return ops->set_vq_affinity(vdpa, index, cpu_mask);
417
418 return 0;
419}
420
421static const struct cpumask *
422virtio_vdpa_get_vq_affinity(struct virtio_device *vdev, int index)
423{
424 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
425 const struct vdpa_config_ops *ops = vdpa->config;
426
427 if (ops->get_vq_affinity)
428 return ops->get_vq_affinity(vdpa, index);
429
430 return NULL;
431}
432
433static const struct virtio_config_ops virtio_vdpa_config_ops = {
434 .get = virtio_vdpa_get,
435 .set = virtio_vdpa_set,
436 .generation = virtio_vdpa_generation,
437 .get_status = virtio_vdpa_get_status,
438 .set_status = virtio_vdpa_set_status,
439 .reset = virtio_vdpa_reset,
440 .find_vqs = virtio_vdpa_find_vqs,
441 .del_vqs = virtio_vdpa_del_vqs,
442 .get_features = virtio_vdpa_get_features,
443 .finalize_features = virtio_vdpa_finalize_features,
444 .bus_name = virtio_vdpa_bus_name,
445 .set_vq_affinity = virtio_vdpa_set_vq_affinity,
446 .get_vq_affinity = virtio_vdpa_get_vq_affinity,
447};
448
449static void virtio_vdpa_release_dev(struct device *_d)
450{
451 struct virtio_device *vdev =
452 container_of(_d, struct virtio_device, dev);
453 struct virtio_vdpa_device *vd_dev =
454 container_of(vdev, struct virtio_vdpa_device, vdev);
455
456 kfree(objp: vd_dev);
457}
458
459static int virtio_vdpa_probe(struct vdpa_device *vdpa)
460{
461 const struct vdpa_config_ops *ops = vdpa->config;
462 struct virtio_vdpa_device *vd_dev, *reg_dev = NULL;
463 int ret = -EINVAL;
464
465 vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL);
466 if (!vd_dev)
467 return -ENOMEM;
468
469 vd_dev->vdev.dev.parent = vdpa->map ? &vdpa->dev :
470 vdpa_get_map(vdev: vdpa).dma_dev;
471 vd_dev->vdev.dev.release = virtio_vdpa_release_dev;
472 vd_dev->vdev.config = &virtio_vdpa_config_ops;
473 vd_dev->vdev.map = vdpa->map;
474 vd_dev->vdpa = vdpa;
475
476 vd_dev->vdev.id.device = ops->get_device_id(vdpa);
477 if (vd_dev->vdev.id.device == 0)
478 goto err;
479
480 vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa);
481 ret = register_virtio_device(dev: &vd_dev->vdev);
482 reg_dev = vd_dev;
483 if (ret)
484 goto err;
485
486 vdpa_set_drvdata(vdev: vdpa, data: vd_dev);
487
488 return 0;
489
490err:
491 if (reg_dev)
492 put_device(dev: &vd_dev->vdev.dev);
493 else
494 kfree(objp: vd_dev);
495 return ret;
496}
497
498static void virtio_vdpa_remove(struct vdpa_device *vdpa)
499{
500 struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdev: vdpa);
501
502 unregister_virtio_device(dev: &vd_dev->vdev);
503}
504
505static struct vdpa_driver virtio_vdpa_driver = {
506 .driver = {
507 .name = "virtio_vdpa",
508 },
509 .probe = virtio_vdpa_probe,
510 .remove = virtio_vdpa_remove,
511};
512
513module_vdpa_driver(virtio_vdpa_driver);
514
515MODULE_VERSION(MOD_VERSION);
516MODULE_LICENSE(MOD_LICENSE);
517MODULE_AUTHOR(MOD_AUTHOR);
518MODULE_DESCRIPTION(MOD_DESC);
519

source code of linux/drivers/virtio/virtio_vdpa.c