1/*
2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM
3 *
4 * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * SOFTWARE.
24 *
25 * Authors:
26 * Kevin Tian <kevin.tian@intel.com>
27 * Jike Song <jike.song@intel.com>
28 * Xiaoguang Chen <xiaoguang.chen@intel.com>
29 * Eddie Dong <eddie.dong@intel.com>
30 *
31 * Contributors:
32 * Niu Bing <bing.niu@intel.com>
33 * Zhi Wang <zhi.a.wang@intel.com>
34 */
35
36#include <linux/init.h>
37#include <linux/mm.h>
38#include <linux/kthread.h>
39#include <linux/sched/mm.h>
40#include <linux/types.h>
41#include <linux/list.h>
42#include <linux/rbtree.h>
43#include <linux/spinlock.h>
44#include <linux/eventfd.h>
45#include <linux/mdev.h>
46#include <linux/debugfs.h>
47
48#include <linux/nospec.h>
49
50#include <drm/drm_edid.h>
51#include <drm/drm_print.h>
52
53#include "i915_drv.h"
54#include "intel_gvt.h"
55#include "gvt.h"
56
57MODULE_IMPORT_NS("DMA_BUF");
58MODULE_IMPORT_NS("I915_GVT");
59
60/* helper macros copied from vfio-pci */
61#define VFIO_PCI_OFFSET_SHIFT 40
62#define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT)
63#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
64#define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
65
66#define EDID_BLOB_OFFSET (PAGE_SIZE/2)
67
68#define OPREGION_SIGNATURE "IntelGraphicsMem"
69
70struct vfio_region;
71struct intel_vgpu_regops {
72 size_t (*rw)(struct intel_vgpu *vgpu, char *buf,
73 size_t count, loff_t *ppos, bool iswrite);
74 void (*release)(struct intel_vgpu *vgpu,
75 struct vfio_region *region);
76};
77
78struct vfio_region {
79 u32 type;
80 u32 subtype;
81 size_t size;
82 u32 flags;
83 const struct intel_vgpu_regops *ops;
84 void *data;
85};
86
87struct vfio_edid_region {
88 struct vfio_region_gfx_edid vfio_edid_regs;
89 void *edid_blob;
90};
91
92struct kvmgt_pgfn {
93 gfn_t gfn;
94 struct hlist_node hnode;
95};
96
97struct gvt_dma {
98 struct intel_vgpu *vgpu;
99 struct rb_node gfn_node;
100 struct rb_node dma_addr_node;
101 gfn_t gfn;
102 dma_addr_t dma_addr;
103 unsigned long size;
104 struct kref ref;
105};
106
107#define vfio_dev_to_vgpu(vfio_dev) \
108 container_of((vfio_dev), struct intel_vgpu, vfio_device)
109
110static void kvmgt_page_track_write(gpa_t gpa, const u8 *val, int len,
111 struct kvm_page_track_notifier_node *node);
112static void kvmgt_page_track_remove_region(gfn_t gfn, unsigned long nr_pages,
113 struct kvm_page_track_notifier_node *node);
114
115static ssize_t intel_vgpu_show_description(struct mdev_type *mtype, char *buf)
116{
117 struct intel_vgpu_type *type =
118 container_of(mtype, struct intel_vgpu_type, type);
119
120 return sprintf(buf, fmt: "low_gm_size: %dMB\nhigh_gm_size: %dMB\n"
121 "fence: %d\nresolution: %s\n"
122 "weight: %d\n",
123 BYTES_TO_MB(type->conf->low_mm),
124 BYTES_TO_MB(type->conf->high_mm),
125 type->conf->fence, vgpu_edid_str(id: type->conf->edid),
126 type->conf->weight);
127}
128
129static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
130 unsigned long size)
131{
132 vfio_unpin_pages(device: &vgpu->vfio_device, iova: gfn << PAGE_SHIFT,
133 DIV_ROUND_UP(size, PAGE_SIZE));
134}
135
136/* Pin a normal or compound guest page for dma. */
137static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
138 unsigned long size, struct page **page)
139{
140 int total_pages = DIV_ROUND_UP(size, PAGE_SIZE);
141 struct page *base_page = NULL;
142 int npage;
143 int ret;
144
145 /*
146 * We pin the pages one-by-one to avoid allocating a big array
147 * on stack to hold pfns.
148 */
149 for (npage = 0; npage < total_pages; npage++) {
150 dma_addr_t cur_iova = (gfn + npage) << PAGE_SHIFT;
151 struct page *cur_page;
152
153 ret = vfio_pin_pages(device: &vgpu->vfio_device, iova: cur_iova, npage: 1,
154 IOMMU_READ | IOMMU_WRITE, pages: &cur_page);
155 if (ret != 1) {
156 gvt_vgpu_err("vfio_pin_pages failed for iova %pad, ret %d\n",
157 &cur_iova, ret);
158 goto err;
159 }
160
161 if (npage == 0)
162 base_page = cur_page;
163 else if (page_to_pfn(base_page) + npage != page_to_pfn(cur_page)) {
164 ret = -EINVAL;
165 npage++;
166 goto err;
167 }
168 }
169
170 *page = base_page;
171 return 0;
172err:
173 if (npage)
174 gvt_unpin_guest_page(vgpu, gfn, size: npage * PAGE_SIZE);
175 return ret;
176}
177
178static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,
179 dma_addr_t *dma_addr, unsigned long size)
180{
181 struct device *dev = vgpu->gvt->gt->i915->drm.dev;
182 struct page *page = NULL;
183 int ret;
184
185 ret = gvt_pin_guest_page(vgpu, gfn, size, page: &page);
186 if (ret)
187 return ret;
188
189 /* Setup DMA mapping. */
190 *dma_addr = dma_map_page(dev, page, 0, size, DMA_BIDIRECTIONAL);
191 if (dma_mapping_error(dev, dma_addr: *dma_addr)) {
192 gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n",
193 page_to_pfn(page), ret);
194 gvt_unpin_guest_page(vgpu, gfn, size);
195 return -ENOMEM;
196 }
197
198 return 0;
199}
200
201static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn,
202 dma_addr_t dma_addr, unsigned long size)
203{
204 struct device *dev = vgpu->gvt->gt->i915->drm.dev;
205
206 dma_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL);
207 gvt_unpin_guest_page(vgpu, gfn, size);
208}
209
210static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu,
211 dma_addr_t dma_addr)
212{
213 struct rb_node *node = vgpu->dma_addr_cache.rb_node;
214 struct gvt_dma *itr;
215
216 while (node) {
217 itr = rb_entry(node, struct gvt_dma, dma_addr_node);
218
219 if (dma_addr < itr->dma_addr)
220 node = node->rb_left;
221 else if (dma_addr > itr->dma_addr)
222 node = node->rb_right;
223 else
224 return itr;
225 }
226 return NULL;
227}
228
229static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn)
230{
231 struct rb_node *node = vgpu->gfn_cache.rb_node;
232 struct gvt_dma *itr;
233
234 while (node) {
235 itr = rb_entry(node, struct gvt_dma, gfn_node);
236
237 if (gfn < itr->gfn)
238 node = node->rb_left;
239 else if (gfn > itr->gfn)
240 node = node->rb_right;
241 else
242 return itr;
243 }
244 return NULL;
245}
246
247static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn,
248 dma_addr_t dma_addr, unsigned long size)
249{
250 struct gvt_dma *new, *itr;
251 struct rb_node **link, *parent = NULL;
252
253 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL);
254 if (!new)
255 return -ENOMEM;
256
257 new->vgpu = vgpu;
258 new->gfn = gfn;
259 new->dma_addr = dma_addr;
260 new->size = size;
261 kref_init(kref: &new->ref);
262
263 /* gfn_cache maps gfn to struct gvt_dma. */
264 link = &vgpu->gfn_cache.rb_node;
265 while (*link) {
266 parent = *link;
267 itr = rb_entry(parent, struct gvt_dma, gfn_node);
268
269 if (gfn < itr->gfn)
270 link = &parent->rb_left;
271 else
272 link = &parent->rb_right;
273 }
274 rb_link_node(node: &new->gfn_node, parent, rb_link: link);
275 rb_insert_color(&new->gfn_node, &vgpu->gfn_cache);
276
277 /* dma_addr_cache maps dma addr to struct gvt_dma. */
278 parent = NULL;
279 link = &vgpu->dma_addr_cache.rb_node;
280 while (*link) {
281 parent = *link;
282 itr = rb_entry(parent, struct gvt_dma, dma_addr_node);
283
284 if (dma_addr < itr->dma_addr)
285 link = &parent->rb_left;
286 else
287 link = &parent->rb_right;
288 }
289 rb_link_node(node: &new->dma_addr_node, parent, rb_link: link);
290 rb_insert_color(&new->dma_addr_node, &vgpu->dma_addr_cache);
291
292 vgpu->nr_cache_entries++;
293 return 0;
294}
295
296static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu,
297 struct gvt_dma *entry)
298{
299 rb_erase(&entry->gfn_node, &vgpu->gfn_cache);
300 rb_erase(&entry->dma_addr_node, &vgpu->dma_addr_cache);
301 kfree(objp: entry);
302 vgpu->nr_cache_entries--;
303}
304
305static void gvt_cache_destroy(struct intel_vgpu *vgpu)
306{
307 struct gvt_dma *dma;
308 struct rb_node *node = NULL;
309
310 for (;;) {
311 mutex_lock(&vgpu->cache_lock);
312 node = rb_first(root: &vgpu->gfn_cache);
313 if (!node) {
314 mutex_unlock(lock: &vgpu->cache_lock);
315 break;
316 }
317 dma = rb_entry(node, struct gvt_dma, gfn_node);
318 gvt_dma_unmap_page(vgpu, gfn: dma->gfn, dma_addr: dma->dma_addr, size: dma->size);
319 __gvt_cache_remove_entry(vgpu, entry: dma);
320 mutex_unlock(lock: &vgpu->cache_lock);
321 }
322}
323
324static void gvt_cache_init(struct intel_vgpu *vgpu)
325{
326 vgpu->gfn_cache = RB_ROOT;
327 vgpu->dma_addr_cache = RB_ROOT;
328 vgpu->nr_cache_entries = 0;
329 mutex_init(&vgpu->cache_lock);
330}
331
332static void kvmgt_protect_table_init(struct intel_vgpu *info)
333{
334 hash_init(info->ptable);
335}
336
337static void kvmgt_protect_table_destroy(struct intel_vgpu *info)
338{
339 struct kvmgt_pgfn *p;
340 struct hlist_node *tmp;
341 int i;
342
343 hash_for_each_safe(info->ptable, i, tmp, p, hnode) {
344 hash_del(node: &p->hnode);
345 kfree(objp: p);
346 }
347}
348
349static struct kvmgt_pgfn *
350__kvmgt_protect_table_find(struct intel_vgpu *info, gfn_t gfn)
351{
352 struct kvmgt_pgfn *p, *res = NULL;
353
354 lockdep_assert_held(&info->vgpu_lock);
355
356 hash_for_each_possible(info->ptable, p, hnode, gfn) {
357 if (gfn == p->gfn) {
358 res = p;
359 break;
360 }
361 }
362
363 return res;
364}
365
366static bool kvmgt_gfn_is_write_protected(struct intel_vgpu *info, gfn_t gfn)
367{
368 struct kvmgt_pgfn *p;
369
370 p = __kvmgt_protect_table_find(info, gfn);
371 return !!p;
372}
373
374static void kvmgt_protect_table_add(struct intel_vgpu *info, gfn_t gfn)
375{
376 struct kvmgt_pgfn *p;
377
378 if (kvmgt_gfn_is_write_protected(info, gfn))
379 return;
380
381 p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC);
382 if (WARN(!p, "gfn: 0x%llx\n", gfn))
383 return;
384
385 p->gfn = gfn;
386 hash_add(info->ptable, &p->hnode, gfn);
387}
388
389static void kvmgt_protect_table_del(struct intel_vgpu *info, gfn_t gfn)
390{
391 struct kvmgt_pgfn *p;
392
393 p = __kvmgt_protect_table_find(info, gfn);
394 if (p) {
395 hash_del(node: &p->hnode);
396 kfree(objp: p);
397 }
398}
399
400static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf,
401 size_t count, loff_t *ppos, bool iswrite)
402{
403 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
404 VFIO_PCI_NUM_REGIONS;
405 void *base = vgpu->region[i].data;
406 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
407
408
409 if (pos >= vgpu->region[i].size || iswrite) {
410 gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n");
411 return -EINVAL;
412 }
413 count = min(count, (size_t)(vgpu->region[i].size - pos));
414 memcpy(buf, base + pos, count);
415
416 return count;
417}
418
419static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu,
420 struct vfio_region *region)
421{
422}
423
424static const struct intel_vgpu_regops intel_vgpu_regops_opregion = {
425 .rw = intel_vgpu_reg_rw_opregion,
426 .release = intel_vgpu_reg_release_opregion,
427};
428
429static bool edid_valid(const void *edid, size_t size)
430{
431 const struct drm_edid *drm_edid;
432 bool is_valid;
433
434 drm_edid = drm_edid_alloc(edid, size);
435 is_valid = drm_edid_valid(drm_edid);
436 drm_edid_free(drm_edid);
437
438 return is_valid;
439}
440
441static int handle_edid_regs(struct intel_vgpu *vgpu,
442 struct vfio_edid_region *region, char *buf,
443 size_t count, u16 offset, bool is_write)
444{
445 struct vfio_region_gfx_edid *regs = &region->vfio_edid_regs;
446 unsigned int data;
447
448 if (offset + count > sizeof(*regs))
449 return -EINVAL;
450
451 if (count != 4)
452 return -EINVAL;
453
454 if (is_write) {
455 data = *((unsigned int *)buf);
456 switch (offset) {
457 case offsetof(struct vfio_region_gfx_edid, link_state):
458 if (data == VFIO_DEVICE_GFX_LINK_STATE_UP) {
459 if (!edid_valid(edid: region->edid_blob, EDID_SIZE)) {
460 gvt_vgpu_err("invalid EDID blob\n");
461 return -EINVAL;
462 }
463 intel_vgpu_emulate_hotplug(vgpu, connected: true);
464 } else if (data == VFIO_DEVICE_GFX_LINK_STATE_DOWN)
465 intel_vgpu_emulate_hotplug(vgpu, connected: false);
466 else {
467 gvt_vgpu_err("invalid EDID link state %d\n",
468 regs->link_state);
469 return -EINVAL;
470 }
471 regs->link_state = data;
472 break;
473 case offsetof(struct vfio_region_gfx_edid, edid_size):
474 if (data > regs->edid_max_size) {
475 gvt_vgpu_err("EDID size is bigger than %d!\n",
476 regs->edid_max_size);
477 return -EINVAL;
478 }
479 regs->edid_size = data;
480 break;
481 default:
482 /* read-only regs */
483 gvt_vgpu_err("write read-only EDID region at offset %d\n",
484 offset);
485 return -EPERM;
486 }
487 } else {
488 memcpy(buf, (char *)regs + offset, count);
489 }
490
491 return count;
492}
493
494static int handle_edid_blob(struct vfio_edid_region *region, char *buf,
495 size_t count, u16 offset, bool is_write)
496{
497 if (offset + count > region->vfio_edid_regs.edid_size)
498 return -EINVAL;
499
500 if (is_write)
501 memcpy(region->edid_blob + offset, buf, count);
502 else
503 memcpy(buf, region->edid_blob + offset, count);
504
505 return count;
506}
507
508static size_t intel_vgpu_reg_rw_edid(struct intel_vgpu *vgpu, char *buf,
509 size_t count, loff_t *ppos, bool iswrite)
510{
511 int ret;
512 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
513 VFIO_PCI_NUM_REGIONS;
514 struct vfio_edid_region *region = vgpu->region[i].data;
515 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
516
517 if (pos < region->vfio_edid_regs.edid_offset) {
518 ret = handle_edid_regs(vgpu, region, buf, count, offset: pos, is_write: iswrite);
519 } else {
520 pos -= EDID_BLOB_OFFSET;
521 ret = handle_edid_blob(region, buf, count, offset: pos, is_write: iswrite);
522 }
523
524 if (ret < 0)
525 gvt_vgpu_err("failed to access EDID region\n");
526
527 return ret;
528}
529
530static void intel_vgpu_reg_release_edid(struct intel_vgpu *vgpu,
531 struct vfio_region *region)
532{
533 kfree(objp: region->data);
534}
535
536static const struct intel_vgpu_regops intel_vgpu_regops_edid = {
537 .rw = intel_vgpu_reg_rw_edid,
538 .release = intel_vgpu_reg_release_edid,
539};
540
541static int intel_vgpu_register_reg(struct intel_vgpu *vgpu,
542 unsigned int type, unsigned int subtype,
543 const struct intel_vgpu_regops *ops,
544 size_t size, u32 flags, void *data)
545{
546 struct vfio_region *region;
547
548 region = krealloc(vgpu->region,
549 (vgpu->num_regions + 1) * sizeof(*region),
550 GFP_KERNEL);
551 if (!region)
552 return -ENOMEM;
553
554 vgpu->region = region;
555 vgpu->region[vgpu->num_regions].type = type;
556 vgpu->region[vgpu->num_regions].subtype = subtype;
557 vgpu->region[vgpu->num_regions].ops = ops;
558 vgpu->region[vgpu->num_regions].size = size;
559 vgpu->region[vgpu->num_regions].flags = flags;
560 vgpu->region[vgpu->num_regions].data = data;
561 vgpu->num_regions++;
562 return 0;
563}
564
565int intel_gvt_set_opregion(struct intel_vgpu *vgpu)
566{
567 void *base;
568 int ret;
569
570 /* Each vgpu has its own opregion, although VFIO would create another
571 * one later. This one is used to expose opregion to VFIO. And the
572 * other one created by VFIO later, is used by guest actually.
573 */
574 base = vgpu_opregion(vgpu)->va;
575 if (!base)
576 return -ENOMEM;
577
578 if (memcmp(p: base, OPREGION_SIGNATURE, size: 16)) {
579 memunmap(addr: base);
580 return -EINVAL;
581 }
582
583 ret = intel_vgpu_register_reg(vgpu,
584 PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
585 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION,
586 ops: &intel_vgpu_regops_opregion, INTEL_GVT_OPREGION_SIZE,
587 VFIO_REGION_INFO_FLAG_READ, data: base);
588
589 return ret;
590}
591
592int intel_gvt_set_edid(struct intel_vgpu *vgpu, int port_num)
593{
594 struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);
595 struct vfio_edid_region *base;
596 int ret;
597
598 base = kzalloc(sizeof(*base), GFP_KERNEL);
599 if (!base)
600 return -ENOMEM;
601
602 /* TODO: Add multi-port and EDID extension block support */
603 base->vfio_edid_regs.edid_offset = EDID_BLOB_OFFSET;
604 base->vfio_edid_regs.edid_max_size = EDID_SIZE;
605 base->vfio_edid_regs.edid_size = EDID_SIZE;
606 base->vfio_edid_regs.max_xres = vgpu_edid_xres(id: port->id);
607 base->vfio_edid_regs.max_yres = vgpu_edid_yres(id: port->id);
608 base->edid_blob = port->edid->edid_block;
609
610 ret = intel_vgpu_register_reg(vgpu,
611 VFIO_REGION_TYPE_GFX,
612 VFIO_REGION_SUBTYPE_GFX_EDID,
613 ops: &intel_vgpu_regops_edid, EDID_SIZE,
614 VFIO_REGION_INFO_FLAG_READ |
615 VFIO_REGION_INFO_FLAG_WRITE |
616 VFIO_REGION_INFO_FLAG_CAPS, data: base);
617
618 return ret;
619}
620
621static void intel_vgpu_dma_unmap(struct vfio_device *vfio_dev, u64 iova,
622 u64 length)
623{
624 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
625 struct gvt_dma *entry;
626 u64 iov_pfn = iova >> PAGE_SHIFT;
627 u64 end_iov_pfn = iov_pfn + length / PAGE_SIZE;
628
629 mutex_lock(&vgpu->cache_lock);
630 for (; iov_pfn < end_iov_pfn; iov_pfn++) {
631 entry = __gvt_cache_find_gfn(vgpu, gfn: iov_pfn);
632 if (!entry)
633 continue;
634
635 gvt_dma_unmap_page(vgpu, gfn: entry->gfn, dma_addr: entry->dma_addr,
636 size: entry->size);
637 __gvt_cache_remove_entry(vgpu, entry);
638 }
639 mutex_unlock(lock: &vgpu->cache_lock);
640}
641
642static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu)
643{
644 struct intel_vgpu *itr;
645 int id;
646 bool ret = false;
647
648 mutex_lock(&vgpu->gvt->lock);
649 for_each_active_vgpu(vgpu->gvt, itr, id) {
650 if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, itr->status))
651 continue;
652
653 if (vgpu->vfio_device.kvm == itr->vfio_device.kvm) {
654 ret = true;
655 goto out;
656 }
657 }
658out:
659 mutex_unlock(lock: &vgpu->gvt->lock);
660 return ret;
661}
662
663static int intel_vgpu_open_device(struct vfio_device *vfio_dev)
664{
665 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
666 int ret;
667
668 if (__kvmgt_vgpu_exist(vgpu))
669 return -EEXIST;
670
671 vgpu->track_node.track_write = kvmgt_page_track_write;
672 vgpu->track_node.track_remove_region = kvmgt_page_track_remove_region;
673 ret = kvm_page_track_register_notifier(kvm: vgpu->vfio_device.kvm,
674 n: &vgpu->track_node);
675 if (ret) {
676 gvt_vgpu_err("KVM is required to use Intel vGPU\n");
677 return ret;
678 }
679
680 set_bit(nr: INTEL_VGPU_STATUS_ATTACHED, addr: vgpu->status);
681
682 debugfs_create_ulong(KVMGT_DEBUGFS_FILENAME, mode: 0444, parent: vgpu->debugfs,
683 value: &vgpu->nr_cache_entries);
684
685 intel_gvt_activate_vgpu(vgpu);
686
687 return 0;
688}
689
690static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu)
691{
692 struct eventfd_ctx *trigger;
693
694 trigger = vgpu->msi_trigger;
695 if (trigger) {
696 eventfd_ctx_put(ctx: trigger);
697 vgpu->msi_trigger = NULL;
698 }
699}
700
701static void intel_vgpu_close_device(struct vfio_device *vfio_dev)
702{
703 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
704
705 intel_gvt_release_vgpu(vgpu);
706
707 clear_bit(nr: INTEL_VGPU_STATUS_ATTACHED, addr: vgpu->status);
708
709 debugfs_lookup_and_remove(KVMGT_DEBUGFS_FILENAME, parent: vgpu->debugfs);
710
711 kvm_page_track_unregister_notifier(kvm: vgpu->vfio_device.kvm,
712 n: &vgpu->track_node);
713
714 kvmgt_protect_table_destroy(info: vgpu);
715 gvt_cache_destroy(vgpu);
716
717 WARN_ON(vgpu->nr_cache_entries);
718
719 vgpu->gfn_cache = RB_ROOT;
720 vgpu->dma_addr_cache = RB_ROOT;
721
722 intel_vgpu_release_msi_eventfd_ctx(vgpu);
723}
724
725static u64 intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar)
726{
727 u32 start_lo, start_hi;
728 u32 mem_type;
729
730 start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
731 PCI_BASE_ADDRESS_MEM_MASK;
732 mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
733 PCI_BASE_ADDRESS_MEM_TYPE_MASK;
734
735 switch (mem_type) {
736 case PCI_BASE_ADDRESS_MEM_TYPE_64:
737 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space
738 + bar + 4));
739 break;
740 case PCI_BASE_ADDRESS_MEM_TYPE_32:
741 case PCI_BASE_ADDRESS_MEM_TYPE_1M:
742 /* 1M mem BAR treated as 32-bit BAR */
743 default:
744 /* mem unknown type treated as 32-bit BAR */
745 start_hi = 0;
746 break;
747 }
748
749 return ((u64)start_hi << 32) | start_lo;
750}
751
752static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, u64 off,
753 void *buf, unsigned int count, bool is_write)
754{
755 u64 bar_start = intel_vgpu_get_bar_addr(vgpu, bar);
756 int ret;
757
758 if (is_write)
759 ret = intel_vgpu_emulate_mmio_write(vgpu,
760 pa: bar_start + off, p_data: buf, bytes: count);
761 else
762 ret = intel_vgpu_emulate_mmio_read(vgpu,
763 pa: bar_start + off, p_data: buf, bytes: count);
764 return ret;
765}
766
767static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, u64 off)
768{
769 return off >= vgpu_aperture_offset(vgpu) &&
770 off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu);
771}
772
773static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, u64 off,
774 void *buf, unsigned long count, bool is_write)
775{
776 void __iomem *aperture_va;
777
778 if (!intel_vgpu_in_aperture(vgpu, off) ||
779 !intel_vgpu_in_aperture(vgpu, off: off + count)) {
780 gvt_vgpu_err("Invalid aperture offset %llu\n", off);
781 return -EINVAL;
782 }
783
784 aperture_va = io_mapping_map_wc(mapping: &vgpu->gvt->gt->ggtt->iomap,
785 ALIGN_DOWN(off, PAGE_SIZE),
786 size: count + offset_in_page(off));
787 if (!aperture_va)
788 return -EIO;
789
790 if (is_write)
791 memcpy_toio(aperture_va + offset_in_page(off), buf, count);
792 else
793 memcpy_fromio(buf, aperture_va + offset_in_page(off), count);
794
795 io_mapping_unmap(vaddr: aperture_va);
796
797 return 0;
798}
799
800static ssize_t intel_vgpu_rw(struct intel_vgpu *vgpu, char *buf,
801 size_t count, loff_t *ppos, bool is_write)
802{
803 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
804 u64 pos = *ppos & VFIO_PCI_OFFSET_MASK;
805 int ret = -EINVAL;
806
807
808 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions) {
809 gvt_vgpu_err("invalid index: %u\n", index);
810 return -EINVAL;
811 }
812
813 switch (index) {
814 case VFIO_PCI_CONFIG_REGION_INDEX:
815 if (is_write)
816 ret = intel_vgpu_emulate_cfg_write(vgpu, offset: pos,
817 p_data: buf, bytes: count);
818 else
819 ret = intel_vgpu_emulate_cfg_read(vgpu, offset: pos,
820 p_data: buf, bytes: count);
821 break;
822 case VFIO_PCI_BAR0_REGION_INDEX:
823 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, off: pos,
824 buf, count, is_write);
825 break;
826 case VFIO_PCI_BAR2_REGION_INDEX:
827 ret = intel_vgpu_aperture_rw(vgpu, off: pos, buf, count, is_write);
828 break;
829 case VFIO_PCI_BAR1_REGION_INDEX:
830 case VFIO_PCI_BAR3_REGION_INDEX:
831 case VFIO_PCI_BAR4_REGION_INDEX:
832 case VFIO_PCI_BAR5_REGION_INDEX:
833 case VFIO_PCI_VGA_REGION_INDEX:
834 case VFIO_PCI_ROM_REGION_INDEX:
835 break;
836 default:
837 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions)
838 return -EINVAL;
839
840 index -= VFIO_PCI_NUM_REGIONS;
841 return vgpu->region[index].ops->rw(vgpu, buf, count,
842 ppos, is_write);
843 }
844
845 return ret == 0 ? count : ret;
846}
847
848static bool gtt_entry(struct intel_vgpu *vgpu, loff_t *ppos)
849{
850 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
851 struct intel_gvt *gvt = vgpu->gvt;
852 int offset;
853
854 /* Only allow MMIO GGTT entry access */
855 if (index != PCI_BASE_ADDRESS_0)
856 return false;
857
858 offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) -
859 intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0);
860
861 return (offset >= gvt->device_info.gtt_start_offset &&
862 offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ?
863 true : false;
864}
865
866static ssize_t intel_vgpu_read(struct vfio_device *vfio_dev, char __user *buf,
867 size_t count, loff_t *ppos)
868{
869 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
870 unsigned int done = 0;
871 int ret;
872
873 while (count) {
874 size_t filled;
875
876 /* Only support GGTT entry 8 bytes read */
877 if (count >= 8 && !(*ppos % 8) &&
878 gtt_entry(vgpu, ppos)) {
879 u64 val;
880
881 ret = intel_vgpu_rw(vgpu, buf: (char *)&val, count: sizeof(val),
882 ppos, is_write: false);
883 if (ret <= 0)
884 goto read_err;
885
886 if (copy_to_user(to: buf, from: &val, n: sizeof(val)))
887 goto read_err;
888
889 filled = 8;
890 } else if (count >= 4 && !(*ppos % 4)) {
891 u32 val;
892
893 ret = intel_vgpu_rw(vgpu, buf: (char *)&val, count: sizeof(val),
894 ppos, is_write: false);
895 if (ret <= 0)
896 goto read_err;
897
898 if (copy_to_user(to: buf, from: &val, n: sizeof(val)))
899 goto read_err;
900
901 filled = 4;
902 } else if (count >= 2 && !(*ppos % 2)) {
903 u16 val;
904
905 ret = intel_vgpu_rw(vgpu, buf: (char *)&val, count: sizeof(val),
906 ppos, is_write: false);
907 if (ret <= 0)
908 goto read_err;
909
910 if (copy_to_user(to: buf, from: &val, n: sizeof(val)))
911 goto read_err;
912
913 filled = 2;
914 } else {
915 u8 val;
916
917 ret = intel_vgpu_rw(vgpu, buf: &val, count: sizeof(val), ppos,
918 is_write: false);
919 if (ret <= 0)
920 goto read_err;
921
922 if (copy_to_user(to: buf, from: &val, n: sizeof(val)))
923 goto read_err;
924
925 filled = 1;
926 }
927
928 count -= filled;
929 done += filled;
930 *ppos += filled;
931 buf += filled;
932 }
933
934 return done;
935
936read_err:
937 return -EFAULT;
938}
939
940static ssize_t intel_vgpu_write(struct vfio_device *vfio_dev,
941 const char __user *buf,
942 size_t count, loff_t *ppos)
943{
944 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
945 unsigned int done = 0;
946 int ret;
947
948 while (count) {
949 size_t filled;
950
951 /* Only support GGTT entry 8 bytes write */
952 if (count >= 8 && !(*ppos % 8) &&
953 gtt_entry(vgpu, ppos)) {
954 u64 val;
955
956 if (copy_from_user(to: &val, from: buf, n: sizeof(val)))
957 goto write_err;
958
959 ret = intel_vgpu_rw(vgpu, buf: (char *)&val, count: sizeof(val),
960 ppos, is_write: true);
961 if (ret <= 0)
962 goto write_err;
963
964 filled = 8;
965 } else if (count >= 4 && !(*ppos % 4)) {
966 u32 val;
967
968 if (copy_from_user(to: &val, from: buf, n: sizeof(val)))
969 goto write_err;
970
971 ret = intel_vgpu_rw(vgpu, buf: (char *)&val, count: sizeof(val),
972 ppos, is_write: true);
973 if (ret <= 0)
974 goto write_err;
975
976 filled = 4;
977 } else if (count >= 2 && !(*ppos % 2)) {
978 u16 val;
979
980 if (copy_from_user(to: &val, from: buf, n: sizeof(val)))
981 goto write_err;
982
983 ret = intel_vgpu_rw(vgpu, buf: (char *)&val,
984 count: sizeof(val), ppos, is_write: true);
985 if (ret <= 0)
986 goto write_err;
987
988 filled = 2;
989 } else {
990 u8 val;
991
992 if (copy_from_user(to: &val, from: buf, n: sizeof(val)))
993 goto write_err;
994
995 ret = intel_vgpu_rw(vgpu, buf: &val, count: sizeof(val),
996 ppos, is_write: true);
997 if (ret <= 0)
998 goto write_err;
999
1000 filled = 1;
1001 }
1002
1003 count -= filled;
1004 done += filled;
1005 *ppos += filled;
1006 buf += filled;
1007 }
1008
1009 return done;
1010write_err:
1011 return -EFAULT;
1012}
1013
1014static int intel_vgpu_mmap(struct vfio_device *vfio_dev,
1015 struct vm_area_struct *vma)
1016{
1017 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1018 unsigned int index;
1019 u64 virtaddr;
1020 unsigned long req_size, pgoff, req_start;
1021 pgprot_t pg_prot;
1022
1023 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1024 if (index >= VFIO_PCI_ROM_REGION_INDEX)
1025 return -EINVAL;
1026
1027 if (vma->vm_end < vma->vm_start)
1028 return -EINVAL;
1029 if ((vma->vm_flags & VM_SHARED) == 0)
1030 return -EINVAL;
1031 if (index != VFIO_PCI_BAR2_REGION_INDEX)
1032 return -EINVAL;
1033
1034 pg_prot = vma->vm_page_prot;
1035 virtaddr = vma->vm_start;
1036 req_size = vma->vm_end - vma->vm_start;
1037 pgoff = vma->vm_pgoff &
1038 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1039 req_start = pgoff << PAGE_SHIFT;
1040
1041 if (!intel_vgpu_in_aperture(vgpu, off: req_start))
1042 return -EINVAL;
1043 if (req_start + req_size >
1044 vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu))
1045 return -EINVAL;
1046
1047 pgoff = (gvt_aperture_pa_base(vgpu->gvt) >> PAGE_SHIFT) + pgoff;
1048
1049 return remap_pfn_range(vma, addr: virtaddr, pfn: pgoff, size: req_size, pgprot: pg_prot);
1050}
1051
1052static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type)
1053{
1054 if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX)
1055 return 1;
1056
1057 return 0;
1058}
1059
1060static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu,
1061 unsigned int index, unsigned int start,
1062 unsigned int count, u32 flags,
1063 void *data)
1064{
1065 return 0;
1066}
1067
1068static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu,
1069 unsigned int index, unsigned int start,
1070 unsigned int count, u32 flags, void *data)
1071{
1072 return 0;
1073}
1074
1075static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu,
1076 unsigned int index, unsigned int start, unsigned int count,
1077 u32 flags, void *data)
1078{
1079 return 0;
1080}
1081
1082static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu,
1083 unsigned int index, unsigned int start, unsigned int count,
1084 u32 flags, void *data)
1085{
1086 struct eventfd_ctx *trigger;
1087
1088 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
1089 int fd = *(int *)data;
1090
1091 trigger = eventfd_ctx_fdget(fd);
1092 if (IS_ERR(ptr: trigger)) {
1093 gvt_vgpu_err("eventfd_ctx_fdget failed\n");
1094 return PTR_ERR(ptr: trigger);
1095 }
1096 vgpu->msi_trigger = trigger;
1097 } else if ((flags & VFIO_IRQ_SET_DATA_NONE) && !count)
1098 intel_vgpu_release_msi_eventfd_ctx(vgpu);
1099
1100 return 0;
1101}
1102
1103static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, u32 flags,
1104 unsigned int index, unsigned int start, unsigned int count,
1105 void *data)
1106{
1107 int (*func)(struct intel_vgpu *vgpu, unsigned int index,
1108 unsigned int start, unsigned int count, u32 flags,
1109 void *data) = NULL;
1110
1111 switch (index) {
1112 case VFIO_PCI_INTX_IRQ_INDEX:
1113 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1114 case VFIO_IRQ_SET_ACTION_MASK:
1115 func = intel_vgpu_set_intx_mask;
1116 break;
1117 case VFIO_IRQ_SET_ACTION_UNMASK:
1118 func = intel_vgpu_set_intx_unmask;
1119 break;
1120 case VFIO_IRQ_SET_ACTION_TRIGGER:
1121 func = intel_vgpu_set_intx_trigger;
1122 break;
1123 }
1124 break;
1125 case VFIO_PCI_MSI_IRQ_INDEX:
1126 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1127 case VFIO_IRQ_SET_ACTION_MASK:
1128 case VFIO_IRQ_SET_ACTION_UNMASK:
1129 /* XXX Need masking support exported */
1130 break;
1131 case VFIO_IRQ_SET_ACTION_TRIGGER:
1132 func = intel_vgpu_set_msi_trigger;
1133 break;
1134 }
1135 break;
1136 }
1137
1138 if (!func)
1139 return -ENOTTY;
1140
1141 return func(vgpu, index, start, count, flags, data);
1142}
1143
1144static int intel_vgpu_ioctl_get_region_info(struct vfio_device *vfio_dev,
1145 struct vfio_region_info *info,
1146 struct vfio_info_cap *caps)
1147{
1148 struct vfio_region_info_cap_sparse_mmap *sparse = NULL;
1149 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1150 int nr_areas = 1;
1151 int cap_type_id;
1152 unsigned int i;
1153 int ret;
1154
1155 switch (info->index) {
1156 case VFIO_PCI_CONFIG_REGION_INDEX:
1157 info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1158 info->size = vgpu->gvt->device_info.cfg_space_size;
1159 info->flags = VFIO_REGION_INFO_FLAG_READ |
1160 VFIO_REGION_INFO_FLAG_WRITE;
1161 break;
1162 case VFIO_PCI_BAR0_REGION_INDEX:
1163 info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1164 info->size = vgpu->cfg_space.bar[info->index].size;
1165 if (!info->size) {
1166 info->flags = 0;
1167 break;
1168 }
1169
1170 info->flags = VFIO_REGION_INFO_FLAG_READ |
1171 VFIO_REGION_INFO_FLAG_WRITE;
1172 break;
1173 case VFIO_PCI_BAR1_REGION_INDEX:
1174 info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1175 info->size = 0;
1176 info->flags = 0;
1177 break;
1178 case VFIO_PCI_BAR2_REGION_INDEX:
1179 info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1180 info->flags = VFIO_REGION_INFO_FLAG_CAPS |
1181 VFIO_REGION_INFO_FLAG_MMAP |
1182 VFIO_REGION_INFO_FLAG_READ |
1183 VFIO_REGION_INFO_FLAG_WRITE;
1184 info->size = gvt_aperture_sz(vgpu->gvt);
1185
1186 sparse = kzalloc(struct_size(sparse, areas, nr_areas),
1187 GFP_KERNEL);
1188 if (!sparse)
1189 return -ENOMEM;
1190
1191 sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1192 sparse->header.version = 1;
1193 sparse->nr_areas = nr_areas;
1194 cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1195 sparse->areas[0].offset =
1196 PAGE_ALIGN(vgpu_aperture_offset(vgpu));
1197 sparse->areas[0].size = vgpu_aperture_sz(vgpu);
1198 break;
1199
1200 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1201 info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1202 info->size = 0;
1203 info->flags = 0;
1204
1205 gvt_dbg_core("get region info bar:%d\n", info->index);
1206 break;
1207
1208 case VFIO_PCI_ROM_REGION_INDEX:
1209 case VFIO_PCI_VGA_REGION_INDEX:
1210 info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1211 info->size = 0;
1212 info->flags = 0;
1213
1214 gvt_dbg_core("get region info index:%d\n", info->index);
1215 break;
1216 default: {
1217 struct vfio_region_info_cap_type cap_type = {
1218 .header.id = VFIO_REGION_INFO_CAP_TYPE,
1219 .header.version = 1
1220 };
1221
1222 if (info->index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions)
1223 return -EINVAL;
1224 info->index = array_index_nospec(
1225 info->index, VFIO_PCI_NUM_REGIONS + vgpu->num_regions);
1226
1227 i = info->index - VFIO_PCI_NUM_REGIONS;
1228
1229 info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1230 info->size = vgpu->region[i].size;
1231 info->flags = vgpu->region[i].flags;
1232
1233 cap_type.type = vgpu->region[i].type;
1234 cap_type.subtype = vgpu->region[i].subtype;
1235
1236 ret = vfio_info_add_capability(caps, cap: &cap_type.header,
1237 size: sizeof(cap_type));
1238 if (ret)
1239 return ret;
1240 }
1241 }
1242
1243 if ((info->flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) {
1244 ret = -EINVAL;
1245 if (cap_type_id == VFIO_REGION_INFO_CAP_SPARSE_MMAP) {
1246 ret = vfio_info_add_capability(
1247 caps, cap: &sparse->header,
1248 struct_size(sparse, areas, sparse->nr_areas));
1249 }
1250 if (ret) {
1251 kfree(objp: sparse);
1252 return ret;
1253 }
1254 }
1255
1256 kfree(objp: sparse);
1257 return 0;
1258}
1259
1260static long intel_vgpu_ioctl(struct vfio_device *vfio_dev, unsigned int cmd,
1261 unsigned long arg)
1262{
1263 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1264 unsigned long minsz;
1265
1266 gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd);
1267
1268 if (cmd == VFIO_DEVICE_GET_INFO) {
1269 struct vfio_device_info info;
1270
1271 minsz = offsetofend(struct vfio_device_info, num_irqs);
1272
1273 if (copy_from_user(to: &info, from: (void __user *)arg, n: minsz))
1274 return -EFAULT;
1275
1276 if (info.argsz < minsz)
1277 return -EINVAL;
1278
1279 info.flags = VFIO_DEVICE_FLAGS_PCI;
1280 info.flags |= VFIO_DEVICE_FLAGS_RESET;
1281 info.num_regions = VFIO_PCI_NUM_REGIONS +
1282 vgpu->num_regions;
1283 info.num_irqs = VFIO_PCI_NUM_IRQS;
1284
1285 return copy_to_user(to: (void __user *)arg, from: &info, n: minsz) ?
1286 -EFAULT : 0;
1287
1288 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
1289 struct vfio_irq_info info;
1290
1291 minsz = offsetofend(struct vfio_irq_info, count);
1292
1293 if (copy_from_user(to: &info, from: (void __user *)arg, n: minsz))
1294 return -EFAULT;
1295
1296 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
1297 return -EINVAL;
1298
1299 switch (info.index) {
1300 case VFIO_PCI_INTX_IRQ_INDEX:
1301 case VFIO_PCI_MSI_IRQ_INDEX:
1302 break;
1303 default:
1304 return -EINVAL;
1305 }
1306
1307 info.flags = VFIO_IRQ_INFO_EVENTFD;
1308
1309 info.count = intel_vgpu_get_irq_count(vgpu, type: info.index);
1310
1311 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
1312 info.flags |= (VFIO_IRQ_INFO_MASKABLE |
1313 VFIO_IRQ_INFO_AUTOMASKED);
1314 else
1315 info.flags |= VFIO_IRQ_INFO_NORESIZE;
1316
1317 return copy_to_user(to: (void __user *)arg, from: &info, n: minsz) ?
1318 -EFAULT : 0;
1319 } else if (cmd == VFIO_DEVICE_SET_IRQS) {
1320 struct vfio_irq_set hdr;
1321 u8 *data = NULL;
1322 int ret = 0;
1323 size_t data_size = 0;
1324
1325 minsz = offsetofend(struct vfio_irq_set, count);
1326
1327 if (copy_from_user(to: &hdr, from: (void __user *)arg, n: minsz))
1328 return -EFAULT;
1329
1330 if (!is_power_of_2(n: hdr.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) ||
1331 !is_power_of_2(n: hdr.flags & VFIO_IRQ_SET_ACTION_TYPE_MASK))
1332 return -EINVAL;
1333
1334 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {
1335 int max = intel_vgpu_get_irq_count(vgpu, type: hdr.index);
1336
1337 if (!hdr.count)
1338 return -EINVAL;
1339
1340 ret = vfio_set_irqs_validate_and_prepare(hdr: &hdr, num_irqs: max,
1341 max_irq_type: VFIO_PCI_NUM_IRQS, data_size: &data_size);
1342 if (ret) {
1343 gvt_vgpu_err("vfio_set_irqs_validate_and_prepare failed\n");
1344 return ret;
1345 }
1346
1347 data = memdup_user((void __user *)(arg + minsz),
1348 data_size);
1349 if (IS_ERR(ptr: data))
1350 return PTR_ERR(ptr: data);
1351 }
1352
1353 ret = intel_vgpu_set_irqs(vgpu, flags: hdr.flags, index: hdr.index,
1354 start: hdr.start, count: hdr.count, data);
1355 kfree(objp: data);
1356
1357 return ret;
1358 } else if (cmd == VFIO_DEVICE_RESET) {
1359 intel_gvt_reset_vgpu(vgpu);
1360 return 0;
1361 } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) {
1362 struct vfio_device_gfx_plane_info dmabuf = {};
1363 int ret = 0;
1364
1365 minsz = offsetofend(struct vfio_device_gfx_plane_info,
1366 dmabuf_id);
1367 if (copy_from_user(to: &dmabuf, from: (void __user *)arg, n: minsz))
1368 return -EFAULT;
1369 if (dmabuf.argsz < minsz)
1370 return -EINVAL;
1371
1372 ret = intel_vgpu_query_plane(vgpu, args: &dmabuf);
1373 if (ret != 0)
1374 return ret;
1375
1376 return copy_to_user(to: (void __user *)arg, from: &dmabuf, n: minsz) ?
1377 -EFAULT : 0;
1378 } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) {
1379 __u32 dmabuf_id;
1380
1381 if (get_user(dmabuf_id, (__u32 __user *)arg))
1382 return -EFAULT;
1383 return intel_vgpu_get_dmabuf(vgpu, dmabuf_id);
1384 }
1385
1386 return -ENOTTY;
1387}
1388
1389static ssize_t
1390vgpu_id_show(struct device *dev, struct device_attribute *attr,
1391 char *buf)
1392{
1393 struct intel_vgpu *vgpu = dev_get_drvdata(dev);
1394
1395 return sprintf(buf, fmt: "%d\n", vgpu->id);
1396}
1397
1398static DEVICE_ATTR_RO(vgpu_id);
1399
1400static struct attribute *intel_vgpu_attrs[] = {
1401 &dev_attr_vgpu_id.attr,
1402 NULL
1403};
1404
1405static const struct attribute_group intel_vgpu_group = {
1406 .name = "intel_vgpu",
1407 .attrs = intel_vgpu_attrs,
1408};
1409
1410static const struct attribute_group *intel_vgpu_groups[] = {
1411 &intel_vgpu_group,
1412 NULL,
1413};
1414
1415static int intel_vgpu_init_dev(struct vfio_device *vfio_dev)
1416{
1417 struct mdev_device *mdev = to_mdev_device(dev: vfio_dev->dev);
1418 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1419 struct intel_vgpu_type *type =
1420 container_of(mdev->type, struct intel_vgpu_type, type);
1421 int ret;
1422
1423 vgpu->gvt = kdev_to_i915(kdev: mdev->type->parent->dev)->gvt;
1424 ret = intel_gvt_create_vgpu(vgpu, conf: type->conf);
1425 if (ret)
1426 return ret;
1427
1428 kvmgt_protect_table_init(info: vgpu);
1429 gvt_cache_init(vgpu);
1430
1431 return 0;
1432}
1433
1434static void intel_vgpu_release_dev(struct vfio_device *vfio_dev)
1435{
1436 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1437
1438 intel_gvt_destroy_vgpu(vgpu);
1439}
1440
1441static const struct vfio_device_ops intel_vgpu_dev_ops = {
1442 .init = intel_vgpu_init_dev,
1443 .release = intel_vgpu_release_dev,
1444 .open_device = intel_vgpu_open_device,
1445 .close_device = intel_vgpu_close_device,
1446 .read = intel_vgpu_read,
1447 .write = intel_vgpu_write,
1448 .mmap = intel_vgpu_mmap,
1449 .ioctl = intel_vgpu_ioctl,
1450 .get_region_info_caps = intel_vgpu_ioctl_get_region_info,
1451 .dma_unmap = intel_vgpu_dma_unmap,
1452 .bind_iommufd = vfio_iommufd_emulated_bind,
1453 .unbind_iommufd = vfio_iommufd_emulated_unbind,
1454 .attach_ioas = vfio_iommufd_emulated_attach_ioas,
1455 .detach_ioas = vfio_iommufd_emulated_detach_ioas,
1456};
1457
1458static int intel_vgpu_probe(struct mdev_device *mdev)
1459{
1460 struct intel_vgpu *vgpu;
1461 int ret;
1462
1463 vgpu = vfio_alloc_device(intel_vgpu, vfio_device, &mdev->dev,
1464 &intel_vgpu_dev_ops);
1465 if (IS_ERR(ptr: vgpu)) {
1466 gvt_err("failed to create intel vgpu: %ld\n", PTR_ERR(vgpu));
1467 return PTR_ERR(ptr: vgpu);
1468 }
1469
1470 dev_set_drvdata(dev: &mdev->dev, data: vgpu);
1471 ret = vfio_register_emulated_iommu_dev(device: &vgpu->vfio_device);
1472 if (ret)
1473 goto out_put_vdev;
1474
1475 gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n",
1476 dev_name(mdev_dev(mdev)));
1477 return 0;
1478
1479out_put_vdev:
1480 vfio_put_device(device: &vgpu->vfio_device);
1481 return ret;
1482}
1483
1484static void intel_vgpu_remove(struct mdev_device *mdev)
1485{
1486 struct intel_vgpu *vgpu = dev_get_drvdata(dev: &mdev->dev);
1487
1488 vfio_unregister_group_dev(device: &vgpu->vfio_device);
1489 vfio_put_device(device: &vgpu->vfio_device);
1490}
1491
1492static unsigned int intel_vgpu_get_available(struct mdev_type *mtype)
1493{
1494 struct intel_vgpu_type *type =
1495 container_of(mtype, struct intel_vgpu_type, type);
1496 struct intel_gvt *gvt = kdev_to_i915(kdev: mtype->parent->dev)->gvt;
1497 unsigned int low_gm_avail, high_gm_avail, fence_avail;
1498
1499 mutex_lock(&gvt->lock);
1500 low_gm_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE -
1501 gvt->gm.vgpu_allocated_low_gm_size;
1502 high_gm_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE -
1503 gvt->gm.vgpu_allocated_high_gm_size;
1504 fence_avail = gvt_fence_sz(gvt) - HOST_FENCE -
1505 gvt->fence.vgpu_allocated_fence_num;
1506 mutex_unlock(lock: &gvt->lock);
1507
1508 return min3(low_gm_avail / type->conf->low_mm,
1509 high_gm_avail / type->conf->high_mm,
1510 fence_avail / type->conf->fence);
1511}
1512
1513static struct mdev_driver intel_vgpu_mdev_driver = {
1514 .device_api = VFIO_DEVICE_API_PCI_STRING,
1515 .driver = {
1516 .name = "intel_vgpu_mdev",
1517 .owner = THIS_MODULE,
1518 .dev_groups = intel_vgpu_groups,
1519 },
1520 .probe = intel_vgpu_probe,
1521 .remove = intel_vgpu_remove,
1522 .get_available = intel_vgpu_get_available,
1523 .show_description = intel_vgpu_show_description,
1524};
1525
1526int intel_gvt_page_track_add(struct intel_vgpu *info, u64 gfn)
1527{
1528 int r;
1529
1530 if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, info->status))
1531 return -ESRCH;
1532
1533 if (kvmgt_gfn_is_write_protected(info, gfn))
1534 return 0;
1535
1536 r = kvm_write_track_add_gfn(kvm: info->vfio_device.kvm, gfn);
1537 if (r)
1538 return r;
1539
1540 kvmgt_protect_table_add(info, gfn);
1541 return 0;
1542}
1543
1544int intel_gvt_page_track_remove(struct intel_vgpu *info, u64 gfn)
1545{
1546 int r;
1547
1548 if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, info->status))
1549 return -ESRCH;
1550
1551 if (!kvmgt_gfn_is_write_protected(info, gfn))
1552 return 0;
1553
1554 r = kvm_write_track_remove_gfn(kvm: info->vfio_device.kvm, gfn);
1555 if (r)
1556 return r;
1557
1558 kvmgt_protect_table_del(info, gfn);
1559 return 0;
1560}
1561
1562static void kvmgt_page_track_write(gpa_t gpa, const u8 *val, int len,
1563 struct kvm_page_track_notifier_node *node)
1564{
1565 struct intel_vgpu *info =
1566 container_of(node, struct intel_vgpu, track_node);
1567
1568 mutex_lock(&info->vgpu_lock);
1569
1570 if (kvmgt_gfn_is_write_protected(info, gfn: gpa >> PAGE_SHIFT))
1571 intel_vgpu_page_track_handler(vgpu: info, gpa,
1572 data: (void *)val, bytes: len);
1573
1574 mutex_unlock(lock: &info->vgpu_lock);
1575}
1576
1577static void kvmgt_page_track_remove_region(gfn_t gfn, unsigned long nr_pages,
1578 struct kvm_page_track_notifier_node *node)
1579{
1580 unsigned long i;
1581 struct intel_vgpu *info =
1582 container_of(node, struct intel_vgpu, track_node);
1583
1584 mutex_lock(&info->vgpu_lock);
1585
1586 for (i = 0; i < nr_pages; i++) {
1587 if (kvmgt_gfn_is_write_protected(info, gfn: gfn + i))
1588 kvmgt_protect_table_del(info, gfn: gfn + i);
1589 }
1590
1591 mutex_unlock(lock: &info->vgpu_lock);
1592}
1593
1594void intel_vgpu_detach_regions(struct intel_vgpu *vgpu)
1595{
1596 int i;
1597
1598 if (!vgpu->region)
1599 return;
1600
1601 for (i = 0; i < vgpu->num_regions; i++)
1602 if (vgpu->region[i].ops->release)
1603 vgpu->region[i].ops->release(vgpu,
1604 &vgpu->region[i]);
1605 vgpu->num_regions = 0;
1606 kfree(objp: vgpu->region);
1607 vgpu->region = NULL;
1608}
1609
1610int intel_gvt_dma_map_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
1611 unsigned long size, dma_addr_t *dma_addr)
1612{
1613 struct gvt_dma *entry;
1614 int ret;
1615
1616 if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))
1617 return -EINVAL;
1618
1619 mutex_lock(&vgpu->cache_lock);
1620
1621 entry = __gvt_cache_find_gfn(vgpu, gfn);
1622 if (!entry) {
1623 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);
1624 if (ret)
1625 goto err_unlock;
1626
1627 ret = __gvt_cache_add(vgpu, gfn, dma_addr: *dma_addr, size);
1628 if (ret)
1629 goto err_unmap;
1630 } else if (entry->size != size) {
1631 /* the same gfn with different size: unmap and re-map */
1632 gvt_dma_unmap_page(vgpu, gfn, dma_addr: entry->dma_addr, size: entry->size);
1633 __gvt_cache_remove_entry(vgpu, entry);
1634
1635 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);
1636 if (ret)
1637 goto err_unlock;
1638
1639 ret = __gvt_cache_add(vgpu, gfn, dma_addr: *dma_addr, size);
1640 if (ret)
1641 goto err_unmap;
1642 } else {
1643 kref_get(kref: &entry->ref);
1644 *dma_addr = entry->dma_addr;
1645 }
1646
1647 mutex_unlock(lock: &vgpu->cache_lock);
1648 return 0;
1649
1650err_unmap:
1651 gvt_dma_unmap_page(vgpu, gfn, dma_addr: *dma_addr, size);
1652err_unlock:
1653 mutex_unlock(lock: &vgpu->cache_lock);
1654 return ret;
1655}
1656
1657int intel_gvt_dma_pin_guest_page(struct intel_vgpu *vgpu, dma_addr_t dma_addr)
1658{
1659 struct gvt_dma *entry;
1660 int ret = 0;
1661
1662 if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))
1663 return -EINVAL;
1664
1665 mutex_lock(&vgpu->cache_lock);
1666 entry = __gvt_cache_find_dma_addr(vgpu, dma_addr);
1667 if (entry)
1668 kref_get(kref: &entry->ref);
1669 else
1670 ret = -ENOMEM;
1671 mutex_unlock(lock: &vgpu->cache_lock);
1672
1673 return ret;
1674}
1675
1676static void __gvt_dma_release(struct kref *ref)
1677{
1678 struct gvt_dma *entry = container_of(ref, typeof(*entry), ref);
1679
1680 gvt_dma_unmap_page(vgpu: entry->vgpu, gfn: entry->gfn, dma_addr: entry->dma_addr,
1681 size: entry->size);
1682 __gvt_cache_remove_entry(vgpu: entry->vgpu, entry);
1683}
1684
1685void intel_gvt_dma_unmap_guest_page(struct intel_vgpu *vgpu,
1686 dma_addr_t dma_addr)
1687{
1688 struct gvt_dma *entry;
1689
1690 if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))
1691 return;
1692
1693 mutex_lock(&vgpu->cache_lock);
1694 entry = __gvt_cache_find_dma_addr(vgpu, dma_addr);
1695 if (entry)
1696 kref_put(kref: &entry->ref, release: __gvt_dma_release);
1697 mutex_unlock(lock: &vgpu->cache_lock);
1698}
1699
1700static void init_device_info(struct intel_gvt *gvt)
1701{
1702 struct intel_gvt_device_info *info = &gvt->device_info;
1703 struct pci_dev *pdev = to_pci_dev(gvt->gt->i915->drm.dev);
1704
1705 info->max_support_vgpus = 8;
1706 info->cfg_space_size = PCI_CFG_SPACE_EXP_SIZE;
1707 info->mmio_size = 2 * 1024 * 1024;
1708 info->mmio_bar = 0;
1709 info->gtt_start_offset = 8 * 1024 * 1024;
1710 info->gtt_entry_size = 8;
1711 info->gtt_entry_size_shift = 3;
1712 info->gmadr_bytes_in_cmd = 8;
1713 info->max_surface_size = 36 * 1024 * 1024;
1714 info->msi_cap_offset = pdev->msi_cap;
1715}
1716
1717static void intel_gvt_test_and_emulate_vblank(struct intel_gvt *gvt)
1718{
1719 struct intel_vgpu *vgpu;
1720 int id;
1721
1722 mutex_lock(&gvt->lock);
1723 idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) {
1724 if (test_and_clear_bit(nr: INTEL_GVT_REQUEST_EMULATE_VBLANK + id,
1725 addr: (void *)&gvt->service_request)) {
1726 if (test_bit(INTEL_VGPU_STATUS_ACTIVE, vgpu->status))
1727 intel_vgpu_emulate_vblank(vgpu);
1728 }
1729 }
1730 mutex_unlock(lock: &gvt->lock);
1731}
1732
1733static int gvt_service_thread(void *data)
1734{
1735 struct intel_gvt *gvt = (struct intel_gvt *)data;
1736 int ret;
1737
1738 gvt_dbg_core("service thread start\n");
1739
1740 while (!kthread_should_stop()) {
1741 ret = wait_event_interruptible(gvt->service_thread_wq,
1742 kthread_should_stop() || gvt->service_request);
1743
1744 if (kthread_should_stop())
1745 break;
1746
1747 if (WARN_ONCE(ret, "service thread is waken up by signal.\n"))
1748 continue;
1749
1750 intel_gvt_test_and_emulate_vblank(gvt);
1751
1752 if (test_bit(INTEL_GVT_REQUEST_SCHED,
1753 (void *)&gvt->service_request) ||
1754 test_bit(INTEL_GVT_REQUEST_EVENT_SCHED,
1755 (void *)&gvt->service_request)) {
1756 intel_gvt_schedule(gvt);
1757 }
1758 }
1759
1760 return 0;
1761}
1762
1763static void clean_service_thread(struct intel_gvt *gvt)
1764{
1765 kthread_stop(k: gvt->service_thread);
1766}
1767
1768static int init_service_thread(struct intel_gvt *gvt)
1769{
1770 init_waitqueue_head(&gvt->service_thread_wq);
1771
1772 gvt->service_thread = kthread_run(gvt_service_thread,
1773 gvt, "gvt_service_thread");
1774 if (IS_ERR(ptr: gvt->service_thread)) {
1775 gvt_err("fail to start service thread.\n");
1776 return PTR_ERR(ptr: gvt->service_thread);
1777 }
1778 return 0;
1779}
1780
1781/**
1782 * intel_gvt_clean_device - clean a GVT device
1783 * @i915: i915 private
1784 *
1785 * This function is called at the driver unloading stage, to free the
1786 * resources owned by a GVT device.
1787 *
1788 */
1789static void intel_gvt_clean_device(struct drm_i915_private *i915)
1790{
1791 struct intel_gvt *gvt = fetch_and_zero(&i915->gvt);
1792
1793 if (drm_WARN_ON(&i915->drm, !gvt))
1794 return;
1795
1796 mdev_unregister_parent(parent: &gvt->parent);
1797 intel_gvt_destroy_idle_vgpu(vgpu: gvt->idle_vgpu);
1798 intel_gvt_clean_vgpu_types(gvt);
1799
1800 intel_gvt_debugfs_clean(gvt);
1801 clean_service_thread(gvt);
1802 intel_gvt_clean_cmd_parser(gvt);
1803 intel_gvt_clean_sched_policy(gvt);
1804 intel_gvt_clean_workload_scheduler(gvt);
1805 intel_gvt_clean_gtt(gvt);
1806 intel_gvt_free_firmware(gvt);
1807 intel_gvt_clean_mmio_info(gvt);
1808 idr_destroy(&gvt->vgpu_idr);
1809
1810 kfree(objp: i915->gvt);
1811}
1812
1813/**
1814 * intel_gvt_init_device - initialize a GVT device
1815 * @i915: drm i915 private data
1816 *
1817 * This function is called at the initialization stage, to initialize
1818 * necessary GVT components.
1819 *
1820 * Returns:
1821 * Zero on success, negative error code if failed.
1822 *
1823 */
1824static int intel_gvt_init_device(struct drm_i915_private *i915)
1825{
1826 struct intel_gvt *gvt;
1827 struct intel_vgpu *vgpu;
1828 int ret;
1829
1830 if (drm_WARN_ON(&i915->drm, i915->gvt))
1831 return -EEXIST;
1832
1833 gvt = kzalloc(sizeof(struct intel_gvt), GFP_KERNEL);
1834 if (!gvt)
1835 return -ENOMEM;
1836
1837 gvt_dbg_core("init gvt device\n");
1838
1839 idr_init_base(idr: &gvt->vgpu_idr, base: 1);
1840 spin_lock_init(&gvt->scheduler.mmio_context_lock);
1841 mutex_init(&gvt->lock);
1842 mutex_init(&gvt->sched_lock);
1843 gvt->gt = to_gt(i915);
1844 i915->gvt = gvt;
1845
1846 init_device_info(gvt);
1847
1848 ret = intel_gvt_setup_mmio_info(gvt);
1849 if (ret)
1850 goto out_clean_idr;
1851
1852 intel_gvt_init_engine_mmio_context(gvt);
1853
1854 ret = intel_gvt_load_firmware(gvt);
1855 if (ret)
1856 goto out_clean_mmio_info;
1857
1858 ret = intel_gvt_init_irq(gvt);
1859 if (ret)
1860 goto out_free_firmware;
1861
1862 ret = intel_gvt_init_gtt(gvt);
1863 if (ret)
1864 goto out_free_firmware;
1865
1866 ret = intel_gvt_init_workload_scheduler(gvt);
1867 if (ret)
1868 goto out_clean_gtt;
1869
1870 ret = intel_gvt_init_sched_policy(gvt);
1871 if (ret)
1872 goto out_clean_workload_scheduler;
1873
1874 ret = intel_gvt_init_cmd_parser(gvt);
1875 if (ret)
1876 goto out_clean_sched_policy;
1877
1878 ret = init_service_thread(gvt);
1879 if (ret)
1880 goto out_clean_cmd_parser;
1881
1882 ret = intel_gvt_init_vgpu_types(gvt);
1883 if (ret)
1884 goto out_clean_thread;
1885
1886 vgpu = intel_gvt_create_idle_vgpu(gvt);
1887 if (IS_ERR(ptr: vgpu)) {
1888 ret = PTR_ERR(ptr: vgpu);
1889 gvt_err("failed to create idle vgpu\n");
1890 goto out_clean_types;
1891 }
1892 gvt->idle_vgpu = vgpu;
1893
1894 intel_gvt_debugfs_init(gvt);
1895
1896 ret = mdev_register_parent(parent: &gvt->parent, dev: i915->drm.dev,
1897 mdev_driver: &intel_vgpu_mdev_driver,
1898 types: gvt->mdev_types, nr_types: gvt->num_types);
1899 if (ret)
1900 goto out_destroy_idle_vgpu;
1901
1902 gvt_dbg_core("gvt device initialization is done\n");
1903 return 0;
1904
1905out_destroy_idle_vgpu:
1906 intel_gvt_destroy_idle_vgpu(vgpu: gvt->idle_vgpu);
1907 intel_gvt_debugfs_clean(gvt);
1908out_clean_types:
1909 intel_gvt_clean_vgpu_types(gvt);
1910out_clean_thread:
1911 clean_service_thread(gvt);
1912out_clean_cmd_parser:
1913 intel_gvt_clean_cmd_parser(gvt);
1914out_clean_sched_policy:
1915 intel_gvt_clean_sched_policy(gvt);
1916out_clean_workload_scheduler:
1917 intel_gvt_clean_workload_scheduler(gvt);
1918out_clean_gtt:
1919 intel_gvt_clean_gtt(gvt);
1920out_free_firmware:
1921 intel_gvt_free_firmware(gvt);
1922out_clean_mmio_info:
1923 intel_gvt_clean_mmio_info(gvt);
1924out_clean_idr:
1925 idr_destroy(&gvt->vgpu_idr);
1926 kfree(objp: gvt);
1927 i915->gvt = NULL;
1928 return ret;
1929}
1930
1931static void intel_gvt_pm_resume(struct drm_i915_private *i915)
1932{
1933 struct intel_gvt *gvt = i915->gvt;
1934
1935 intel_gvt_restore_fence(gvt);
1936 intel_gvt_restore_mmio(gvt);
1937 intel_gvt_restore_ggtt(gvt);
1938}
1939
1940static const struct intel_vgpu_ops intel_gvt_vgpu_ops = {
1941 .init_device = intel_gvt_init_device,
1942 .clean_device = intel_gvt_clean_device,
1943 .pm_resume = intel_gvt_pm_resume,
1944};
1945
1946static int __init kvmgt_init(void)
1947{
1948 int ret;
1949
1950 ret = intel_gvt_set_ops(ops: &intel_gvt_vgpu_ops);
1951 if (ret)
1952 return ret;
1953
1954 ret = mdev_register_driver(drv: &intel_vgpu_mdev_driver);
1955 if (ret)
1956 intel_gvt_clear_ops(ops: &intel_gvt_vgpu_ops);
1957 return ret;
1958}
1959
1960static void __exit kvmgt_exit(void)
1961{
1962 mdev_unregister_driver(drv: &intel_vgpu_mdev_driver);
1963 intel_gvt_clear_ops(ops: &intel_gvt_vgpu_ops);
1964}
1965
1966module_init(kvmgt_init);
1967module_exit(kvmgt_exit);
1968
1969MODULE_DESCRIPTION("Intel mediated pass-through framework for KVM");
1970MODULE_LICENSE("GPL and additional rights");
1971MODULE_AUTHOR("Intel Corporation");
1972

source code of linux/drivers/gpu/drm/i915/gvt/kvmgt.c