1	// SPDX-License-Identifier: GPL-2.0-only
2	/******************************************************************************
3	* privcmd.c
4	*
5	* Interface to privileged domain-0 commands.
6	*
7	* Copyright (c) 2002-2004, K A Fraser, B Dragovic
8	*/
9
10	#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
11
12	#include <linux/eventfd.h>
13	#include <linux/file.h>
14	#include <linux/kernel.h>
15	#include <linux/module.h>
16	#include <linux/mutex.h>
17	#include <linux/poll.h>
18	#include <linux/sched.h>
19	#include <linux/slab.h>
20	#include <linux/srcu.h>
21	#include <linux/string.h>
22	#include <linux/workqueue.h>
23	#include <linux/errno.h>
24	#include <linux/mm.h>
25	#include <linux/mman.h>
26	#include <linux/uaccess.h>
27	#include <linux/swap.h>
28	#include <linux/highmem.h>
29	#include <linux/pagemap.h>
30	#include <linux/seq_file.h>
31	#include <linux/miscdevice.h>
32	#include <linux/moduleparam.h>
33	#include <linux/virtio_mmio.h>
34
35	#include <asm/xen/hypervisor.h>
36	#include <asm/xen/hypercall.h>
37
38	#include <xen/xen.h>
39	#include <xen/events.h>
40	#include <xen/privcmd.h>
41	#include <xen/interface/xen.h>
42	#include <xen/interface/memory.h>
43	#include <xen/interface/hvm/dm_op.h>
44	#include <xen/interface/hvm/ioreq.h>
45	#include <xen/features.h>
46	#include <xen/page.h>
47	#include <xen/xen-ops.h>
48	#include <xen/balloon.h>
49	#ifdef CONFIG_XEN_ACPI
50	#include <xen/acpi.h>
51	#endif
52
53	#include "privcmd.h"
54
55	MODULE_DESCRIPTION("Xen hypercall passthrough driver");
56	MODULE_LICENSE("GPL");
57
58	#define PRIV_VMA_LOCKED ((void *)1)
59
60	static unsigned int privcmd_dm_op_max_num = 16;
61	module_param_named(dm_op_max_nr_bufs, privcmd_dm_op_max_num, uint, 0644);
62	MODULE_PARM_DESC(dm_op_max_nr_bufs,
63	"Maximum number of buffers per dm_op hypercall");
64
65	static unsigned int privcmd_dm_op_buf_max_size = 4096;
66	module_param_named(dm_op_buf_max_size, privcmd_dm_op_buf_max_size, uint,
67	0644);
68	MODULE_PARM_DESC(dm_op_buf_max_size,
69	"Maximum size of a dm_op hypercall buffer");
70
71	struct privcmd_data {
72	domid_t domid;
73	};
74
75	static int privcmd_vma_range_is_mapped(
76	struct vm_area_struct *vma,
77	unsigned long addr,
78	unsigned long nr_pages);
79
80	static long privcmd_ioctl_hypercall(struct file *file, void __user *udata)
81	{
82	struct privcmd_data *data = file->private_data;
83	struct privcmd_hypercall hypercall;
84	long ret;
85
86	/* Disallow arbitrary hypercalls if restricted */
87	if (data->domid != DOMID_INVALID)
88	return -EPERM;
89
90	if (copy_from_user(to: &hypercall, from: udata, n: sizeof(hypercall)))
91	return -EFAULT;
92
93	xen_preemptible_hcall_begin();
94	ret = privcmd_call(call: hypercall.op,
95	a1: hypercall.arg[0], a2: hypercall.arg[1],
96	a3: hypercall.arg[2], a4: hypercall.arg[3],
97	a5: hypercall.arg[4]);
98	xen_preemptible_hcall_end();
99
100	return ret;
101	}
102
103	static void free_page_list(struct list_head *pages)
104	{
105	struct page *p, *n;
106
107	list_for_each_entry_safe(p, n, pages, lru)
108	__free_page(p);
109
110	INIT_LIST_HEAD(list: pages);
111	}
112
113	/*
114	* Given an array of items in userspace, return a list of pages
115	* containing the data. If copying fails, either because of memory
116	* allocation failure or a problem reading user memory, return an
117	* error code; its up to the caller to dispose of any partial list.
118	*/
119	static int gather_array(struct list_head *pagelist,
120	unsigned nelem, size_t size,
121	const void __user *data)
122	{
123	unsigned pageidx;
124	void *pagedata;
125	int ret;
126
127	if (size > PAGE_SIZE)
128	return 0;
129
130	pageidx = PAGE_SIZE;
131	pagedata = NULL; /* quiet, gcc */
132	while (nelem--) {
133	if (pageidx > PAGE_SIZE-size) {
134	struct page *page = alloc_page(GFP_KERNEL);
135
136	ret = -ENOMEM;
137	if (page == NULL)
138	goto fail;
139
140	pagedata = page_address(page);
141
142	list_add_tail(new: &page->lru, head: pagelist);
143	pageidx = 0;
144	}
145
146	ret = -EFAULT;
147	if (copy_from_user(to: pagedata + pageidx, from: data, n: size))
148	goto fail;
149
150	data += size;
151	pageidx += size;
152	}
153
154	ret = 0;
155
156	fail:
157	return ret;
158	}
159
160	/*
161	* Call function "fn" on each element of the array fragmented
162	* over a list of pages.
163	*/
164	static int traverse_pages(unsigned nelem, size_t size,
165	struct list_head *pos,
166	int (*fn)(void *data, void *state),
167	void *state)
168	{
169	void *pagedata;
170	unsigned pageidx;
171	int ret = 0;
172
173	BUG_ON(size > PAGE_SIZE);
174
175	pageidx = PAGE_SIZE;
176	pagedata = NULL; /* hush, gcc */
177
178	while (nelem--) {
179	if (pageidx > PAGE_SIZE-size) {
180	struct page *page;
181	pos = pos->next;
182	page = list_entry(pos, struct page, lru);
183	pagedata = page_address(page);
184	pageidx = 0;
185	}
186
187	ret = (*fn)(pagedata + pageidx, state);
188	if (ret)
189	break;
190	pageidx += size;
191	}
192
193	return ret;
194	}
195
196	/*
197	* Similar to traverse_pages, but use each page as a "block" of
198	* data to be processed as one unit.
199	*/
200	static int traverse_pages_block(unsigned nelem, size_t size,
201	struct list_head *pos,
202	int (*fn)(void *data, int nr, void *state),
203	void *state)
204	{
205	void *pagedata;
206	int ret = 0;
207
208	BUG_ON(size > PAGE_SIZE);
209
210	while (nelem) {
211	int nr = (PAGE_SIZE/size);
212	struct page *page;
213	if (nr > nelem)
214	nr = nelem;
215	pos = pos->next;
216	page = list_entry(pos, struct page, lru);
217	pagedata = page_address(page);
218	ret = (*fn)(pagedata, nr, state);
219	if (ret)
220	break;
221	nelem -= nr;
222	}
223
224	return ret;
225	}
226
227	struct mmap_gfn_state {
228	unsigned long va;
229	struct vm_area_struct *vma;
230	domid_t domain;
231	};
232
233	static int mmap_gfn_range(void *data, void *state)
234	{
235	struct privcmd_mmap_entry *msg = data;
236	struct mmap_gfn_state *st = state;
237	struct vm_area_struct *vma = st->vma;
238	int rc;
239
240	/* Do not allow range to wrap the address space. */
241	if ((msg->npages > (LONG_MAX >> PAGE_SHIFT)) ||
242	((unsigned long)(msg->npages << PAGE_SHIFT) >= -st->va))
243	return -EINVAL;
244
245	/* Range chunks must be contiguous in va space. */
246	if ((msg->va != st->va) ||
247	((msg->va+(msg->npages<<PAGE_SHIFT)) > vma->vm_end))
248	return -EINVAL;
249
250	rc = xen_remap_domain_gfn_range(vma,
251	addr: msg->va & PAGE_MASK,
252	gfn: msg->mfn, nr: msg->npages,
253	prot: vma->vm_page_prot,
254	domid: st->domain, NULL);
255	if (rc < 0)
256	return rc;
257
258	st->va += msg->npages << PAGE_SHIFT;
259
260	return 0;
261	}
262
263	static long privcmd_ioctl_mmap(struct file *file, void __user *udata)
264	{
265	struct privcmd_data *data = file->private_data;
266	struct privcmd_mmap mmapcmd;
267	struct mm_struct *mm = current->mm;
268	struct vm_area_struct *vma;
269	int rc;
270	LIST_HEAD(pagelist);
271	struct mmap_gfn_state state;
272
273	/* We only support privcmd_ioctl_mmap_batch for non-auto-translated. */
274	if (!xen_pv_domain())
275	return -ENOSYS;
276
277	if (copy_from_user(to: &mmapcmd, from: udata, n: sizeof(mmapcmd)))
278	return -EFAULT;
279
280	/* If restriction is in place, check the domid matches */
281	if (data->domid != DOMID_INVALID && data->domid != mmapcmd.dom)
282	return -EPERM;
283
284	rc = gather_array(pagelist: &pagelist,
285	nelem: mmapcmd.num, size: sizeof(struct privcmd_mmap_entry),
286	data: mmapcmd.entry);
287
288	if (rc || list_empty(head: &pagelist))
289	goto out;
290
291	mmap_write_lock(mm);
292
293	{
294	struct page *page = list_first_entry(&pagelist,
295	struct page, lru);
296	struct privcmd_mmap_entry *msg = page_address(page);
297
298	vma = vma_lookup(mm, addr: msg->va);
299	rc = -EINVAL;
300
301	if (!vma || (msg->va != vma->vm_start) || vma->vm_private_data)
302	goto out_up;
303	vma->vm_private_data = PRIV_VMA_LOCKED;
304	}
305
306	state.va = vma->vm_start;
307	state.vma = vma;
308	state.domain = mmapcmd.dom;
309
310	rc = traverse_pages(nelem: mmapcmd.num, size: sizeof(struct privcmd_mmap_entry),
311	pos: &pagelist,
312	fn: mmap_gfn_range, state: &state);
313
314
315	out_up:
316	mmap_write_unlock(mm);
317
318	out:
319	free_page_list(pages: &pagelist);
320
321	return rc;
322	}
323
324	struct mmap_batch_state {
325	domid_t domain;
326	unsigned long va;
327	struct vm_area_struct *vma;
328	int index;
329	/* A tristate:
330	* 0 for no errors
331	* 1 if at least one error has happened (and no
332	* -ENOENT errors have happened)
333	* -ENOENT if at least 1 -ENOENT has happened.
334	*/
335	int global_error;
336	int version;
337
338	/* User-space gfn array to store errors in the second pass for V1. */
339	xen_pfn_t __user *user_gfn;
340	/* User-space int array to store errors in the second pass for V2. */
341	int __user *user_err;
342	};
343
344	/* auto translated dom0 note: if domU being created is PV, then gfn is
345	* mfn(addr on bus). If it's auto xlated, then gfn is pfn (input to HAP).
346	*/
347	static int mmap_batch_fn(void *data, int nr, void *state)
348	{
349	xen_pfn_t *gfnp = data;
350	struct mmap_batch_state *st = state;
351	struct vm_area_struct *vma = st->vma;
352	struct page **pages = vma->vm_private_data;
353	struct page **cur_pages = NULL;
354	int ret;
355
356	if (!xen_pv_domain())
357	cur_pages = &pages[st->index];
358
359	BUG_ON(nr < 0);
360	ret = xen_remap_domain_gfn_array(vma: st->vma, addr: st->va & PAGE_MASK, gfn: gfnp, nr,
361	err_ptr: (int *)gfnp, prot: st->vma->vm_page_prot,
362	domid: st->domain, pages: cur_pages);
363
364	/* Adjust the global_error? */
365	if (ret != nr) {
366	if (ret == -ENOENT)
367	st->global_error = -ENOENT;
368	else {
369	/* Record that at least one error has happened. */
370	if (st->global_error == 0)
371	st->global_error = 1;
372	}
373	}
374	st->va += XEN_PAGE_SIZE * nr;
375	st->index += nr / XEN_PFN_PER_PAGE;
376
377	return 0;
378	}
379
380	static int mmap_return_error(int err, struct mmap_batch_state *st)
381	{
382	int ret;
383
384	if (st->version == 1) {
385	if (err) {
386	xen_pfn_t gfn;
387
388	ret = get_user(gfn, st->user_gfn);
389	if (ret < 0)
390	return ret;
391	/*
392	* V1 encodes the error codes in the 32bit top
393	* nibble of the gfn (with its known
394	* limitations vis-a-vis 64 bit callers).
395	*/
396	gfn |= (err == -ENOENT) ?
397	PRIVCMD_MMAPBATCH_PAGED_ERROR :
398	PRIVCMD_MMAPBATCH_MFN_ERROR;
399	return __put_user(gfn, st->user_gfn++);
400	} else
401	st->user_gfn++;
402	} else { /* st->version == 2 */
403	if (err)
404	return __put_user(err, st->user_err++);
405	else
406	st->user_err++;
407	}
408
409	return 0;
410	}
411
412	static int mmap_return_errors(void *data, int nr, void *state)
413	{
414	struct mmap_batch_state *st = state;
415	int *errs = data;
416	int i;
417	int ret;
418
419	for (i = 0; i < nr; i++) {
420	ret = mmap_return_error(err: errs[i], st);
421	if (ret < 0)
422	return ret;
423	}
424	return 0;
425	}
426
427	/* Allocate pfns that are then mapped with gfns from foreign domid. Update
428	* the vma with the page info to use later.
429	* Returns: 0 if success, otherwise -errno
430	*/
431	static int alloc_empty_pages(struct vm_area_struct *vma, int numpgs)
432	{
433	int rc;
434	struct page **pages;
435
436	pages = kvcalloc(numpgs, sizeof(pages[0]), GFP_KERNEL);
437	if (pages == NULL)
438	return -ENOMEM;
439
440	rc = xen_alloc_unpopulated_pages(nr_pages: numpgs, pages);
441	if (rc != 0) {
442	pr_warn("%s Could not alloc %d pfns rc:%d\n", __func__,
443	numpgs, rc);
444	kvfree(addr: pages);
445	return -ENOMEM;
446	}
447	BUG_ON(vma->vm_private_data != NULL);
448	vma->vm_private_data = pages;
449
450	return 0;
451	}
452
453	static const struct vm_operations_struct privcmd_vm_ops;
454
455	static long privcmd_ioctl_mmap_batch(
456	struct file *file, void __user *udata, int version)
457	{
458	struct privcmd_data *data = file->private_data;
459	int ret;
460	struct privcmd_mmapbatch_v2 m;
461	struct mm_struct *mm = current->mm;
462	struct vm_area_struct *vma;
463	unsigned long nr_pages;
464	LIST_HEAD(pagelist);
465	struct mmap_batch_state state;
466
467	switch (version) {
468	case 1:
469	if (copy_from_user(to: &m, from: udata, n: sizeof(struct privcmd_mmapbatch)))
470	return -EFAULT;
471	/* Returns per-frame error in m.arr. */
472	m.err = NULL;
473	if (!access_ok(m.arr, m.num * sizeof(*m.arr)))
474	return -EFAULT;
475	break;
476	case 2:
477	if (copy_from_user(to: &m, from: udata, n: sizeof(struct privcmd_mmapbatch_v2)))
478	return -EFAULT;
479	/* Returns per-frame error code in m.err. */
480	if (!access_ok(m.err, m.num * (sizeof(*m.err))))
481	return -EFAULT;
482	break;
483	default:
484	return -EINVAL;
485	}
486
487	/* If restriction is in place, check the domid matches */
488	if (data->domid != DOMID_INVALID && data->domid != m.dom)
489	return -EPERM;
490
491	nr_pages = DIV_ROUND_UP(m.num, XEN_PFN_PER_PAGE);
492	if ((m.num <= 0) || (nr_pages > (LONG_MAX >> PAGE_SHIFT)))
493	return -EINVAL;
494
495	ret = gather_array(pagelist: &pagelist, nelem: m.num, size: sizeof(xen_pfn_t), data: m.arr);
496
497	if (ret)
498	goto out;
499	if (list_empty(head: &pagelist)) {
500	ret = -EINVAL;
501	goto out;
502	}
503
504	if (version == 2) {
505	/* Zero error array now to only copy back actual errors. */
506	if (clear_user(to: m.err, n: sizeof(int) * m.num)) {
507	ret = -EFAULT;
508	goto out;
509	}
510	}
511
512	mmap_write_lock(mm);
513
514	vma = find_vma(mm, addr: m.addr);
515	if (!vma ||
516	vma->vm_ops != &privcmd_vm_ops) {
517	ret = -EINVAL;
518	goto out_unlock;
519	}
520
521	/*
522	* Caller must either:
523	*
524	* Map the whole VMA range, which will also allocate all the
525	* pages required for the auto_translated_physmap case.
526	*
527	* Or
528	*
529	* Map unmapped holes left from a previous map attempt (e.g.,
530	* because those foreign frames were previously paged out).
531	*/
532	if (vma->vm_private_data == NULL) {
533	if (m.addr != vma->vm_start ||
534	m.addr + (nr_pages << PAGE_SHIFT) != vma->vm_end) {
535	ret = -EINVAL;
536	goto out_unlock;
537	}
538	if (!xen_pv_domain()) {
539	ret = alloc_empty_pages(vma, numpgs: nr_pages);
540	if (ret < 0)
541	goto out_unlock;
542	} else
543	vma->vm_private_data = PRIV_VMA_LOCKED;
544	} else {
545	if (m.addr < vma->vm_start ||
546	m.addr + (nr_pages << PAGE_SHIFT) > vma->vm_end) {
547	ret = -EINVAL;
548	goto out_unlock;
549	}
550	if (privcmd_vma_range_is_mapped(vma, addr: m.addr, nr_pages)) {
551	ret = -EINVAL;
552	goto out_unlock;
553	}
554	}
555
556	state.domain = m.dom;
557	state.vma = vma;
558	state.va = m.addr;
559	state.index = 0;
560	state.global_error = 0;
561	state.version = version;
562
563	BUILD_BUG_ON(((PAGE_SIZE / sizeof(xen_pfn_t)) % XEN_PFN_PER_PAGE) != 0);
564	/* mmap_batch_fn guarantees ret == 0 */
565	BUG_ON(traverse_pages_block(m.num, sizeof(xen_pfn_t),
566	&pagelist, mmap_batch_fn, &state));
567
568	mmap_write_unlock(mm);
569
570	if (state.global_error) {
571	/* Write back errors in second pass. */
572	state.user_gfn = (xen_pfn_t *)m.arr;
573	state.user_err = m.err;
574	ret = traverse_pages_block(nelem: m.num, size: sizeof(xen_pfn_t),
575	pos: &pagelist, fn: mmap_return_errors, state: &state);
576	} else
577	ret = 0;
578
579	/* If we have not had any EFAULT-like global errors then set the global
580	* error to -ENOENT if necessary. */
581	if ((ret == 0) && (state.global_error == -ENOENT))
582	ret = -ENOENT;
583
584	out:
585	free_page_list(pages: &pagelist);
586	return ret;
587
588	out_unlock:
589	mmap_write_unlock(mm);
590	goto out;
591	}
592
593	static int lock_pages(
594	struct privcmd_dm_op_buf kbufs[], unsigned int num,
595	struct page *pages[], unsigned int nr_pages, unsigned int *pinned)
596	{
597	unsigned int i, off = 0;
598
599	for (i = 0; i < num; ) {
600	unsigned int requested;
601	int page_count;
602
603	requested = DIV_ROUND_UP(
604	offset_in_page(kbufs[i].uptr) + kbufs[i].size,
605	PAGE_SIZE) - off;
606	if (requested > nr_pages)
607	return -ENOSPC;
608
609	page_count = pin_user_pages_fast(
610	start: (unsigned long)kbufs[i].uptr + off * PAGE_SIZE,
611	nr_pages: requested, gup_flags: FOLL_WRITE, pages);
612	if (page_count <= 0)
613	return page_count ? : -EFAULT;
614
615	*pinned += page_count;
616	nr_pages -= page_count;
617	pages += page_count;
618
619	off = (requested == page_count) ? 0 : off + page_count;
620	i += !off;
621	}
622
623	return 0;
624	}
625
626	static void unlock_pages(struct page *pages[], unsigned int nr_pages)
627	{
628	unpin_user_pages_dirty_lock(pages, npages: nr_pages, make_dirty: true);
629	}
630
631	static long privcmd_ioctl_dm_op(struct file *file, void __user *udata)
632	{
633	struct privcmd_data *data = file->private_data;
634	struct privcmd_dm_op kdata;
635	struct privcmd_dm_op_buf *kbufs;
636	unsigned int nr_pages = 0;
637	struct page **pages = NULL;
638	struct xen_dm_op_buf *xbufs = NULL;
639	unsigned int i;
640	long rc;
641	unsigned int pinned = 0;
642
643	if (copy_from_user(to: &kdata, from: udata, n: sizeof(kdata)))
644	return -EFAULT;
645
646	/* If restriction is in place, check the domid matches */
647	if (data->domid != DOMID_INVALID && data->domid != kdata.dom)
648	return -EPERM;
649
650	if (kdata.num == 0)
651	return 0;
652
653	if (kdata.num > privcmd_dm_op_max_num)
654	return -E2BIG;
655
656	kbufs = kcalloc(kdata.num, sizeof(*kbufs), GFP_KERNEL);
657	if (!kbufs)
658	return -ENOMEM;
659
660	if (copy_from_user(to: kbufs, from: kdata.ubufs,
661	n: sizeof(*kbufs) * kdata.num)) {
662	rc = -EFAULT;
663	goto out;
664	}
665
666	for (i = 0; i < kdata.num; i++) {
667	if (kbufs[i].size > privcmd_dm_op_buf_max_size) {
668	rc = -E2BIG;
669	goto out;
670	}
671
672	if (!access_ok(kbufs[i].uptr,
673	kbufs[i].size)) {
674	rc = -EFAULT;
675	goto out;
676	}
677
678	nr_pages += DIV_ROUND_UP(
679	offset_in_page(kbufs[i].uptr) + kbufs[i].size,
680	PAGE_SIZE);
681	}
682
683	pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL);
684	if (!pages) {
685	rc = -ENOMEM;
686	goto out;
687	}
688
689	xbufs = kcalloc(kdata.num, sizeof(*xbufs), GFP_KERNEL);
690	if (!xbufs) {
691	rc = -ENOMEM;
692	goto out;
693	}
694
695	rc = lock_pages(kbufs, num: kdata.num, pages, nr_pages, pinned: &pinned);
696	if (rc < 0)
697	goto out;
698
699	for (i = 0; i < kdata.num; i++) {
700	set_xen_guest_handle(xbufs[i].h, kbufs[i].uptr);
701	xbufs[i].size = kbufs[i].size;
702	}
703
704	xen_preemptible_hcall_begin();
705	rc = HYPERVISOR_dm_op(dom: kdata.dom, nr_bufs: kdata.num, bufs: xbufs);
706	xen_preemptible_hcall_end();
707
708	out:
709	unlock_pages(pages, nr_pages: pinned);
710	kfree(objp: xbufs);
711	kfree(objp: pages);
712	kfree(objp: kbufs);
713
714	return rc;
715	}
716
717	static long privcmd_ioctl_restrict(struct file *file, void __user *udata)
718	{
719	struct privcmd_data *data = file->private_data;
720	domid_t dom;
721
722	if (copy_from_user(to: &dom, from: udata, n: sizeof(dom)))
723	return -EFAULT;
724
725	/* Set restriction to the specified domain, or check it matches */
726	if (data->domid == DOMID_INVALID)
727	data->domid = dom;
728	else if (data->domid != dom)
729	return -EINVAL;
730
731	return 0;
732	}
733
734	static long privcmd_ioctl_mmap_resource(struct file *file,
735	struct privcmd_mmap_resource __user *udata)
736	{
737	struct privcmd_data *data = file->private_data;
738	struct mm_struct *mm = current->mm;
739	struct vm_area_struct *vma;
740	struct privcmd_mmap_resource kdata;
741	xen_pfn_t *pfns = NULL;
742	struct xen_mem_acquire_resource xdata = { };
743	int rc;
744
745	if (copy_from_user(to: &kdata, from: udata, n: sizeof(kdata)))
746	return -EFAULT;
747
748	/* If restriction is in place, check the domid matches */
749	if (data->domid != DOMID_INVALID && data->domid != kdata.dom)
750	return -EPERM;
751
752	/* Both fields must be set or unset */
753	if (!!kdata.addr != !!kdata.num)
754	return -EINVAL;
755
756	xdata.domid = kdata.dom;
757	xdata.type = kdata.type;
758	xdata.id = kdata.id;
759
760	if (!kdata.addr && !kdata.num) {
761	/* Query the size of the resource. */
762	rc = HYPERVISOR_memory_op(XENMEM_acquire_resource, arg: &xdata);
763	if (rc)
764	return rc;
765	return __put_user(xdata.nr_frames, &udata->num);
766	}
767
768	mmap_write_lock(mm);
769
770	vma = find_vma(mm, addr: kdata.addr);
771	if (!vma || vma->vm_ops != &privcmd_vm_ops) {
772	rc = -EINVAL;
773	goto out;
774	}
775
776	pfns = kcalloc(kdata.num, sizeof(*pfns), GFP_KERNEL | __GFP_NOWARN);
777	if (!pfns) {
778	rc = -ENOMEM;
779	goto out;
780	}
781
782	if (IS_ENABLED(CONFIG_XEN_AUTO_XLATE) && !xen_pv_domain()) {
783	unsigned int nr = DIV_ROUND_UP(kdata.num, XEN_PFN_PER_PAGE);
784	struct page **pages;
785	unsigned int i;
786
787	rc = alloc_empty_pages(vma, numpgs: nr);
788	if (rc < 0)
789	goto out;
790
791	pages = vma->vm_private_data;
792
793	for (i = 0; i < kdata.num; i++) {
794	xen_pfn_t pfn =
795	page_to_xen_pfn(pages[i / XEN_PFN_PER_PAGE]);
796
797	pfns[i] = pfn + (i % XEN_PFN_PER_PAGE);
798	}
799	} else
800	vma->vm_private_data = PRIV_VMA_LOCKED;
801
802	xdata.frame = kdata.idx;
803	xdata.nr_frames = kdata.num;
804	set_xen_guest_handle(xdata.frame_list, pfns);
805
806	xen_preemptible_hcall_begin();
807	rc = HYPERVISOR_memory_op(XENMEM_acquire_resource, arg: &xdata);
808	xen_preemptible_hcall_end();
809
810	if (rc)
811	goto out;
812
813	if (IS_ENABLED(CONFIG_XEN_AUTO_XLATE) && !xen_pv_domain()) {
814	rc = xen_remap_vma_range(vma, addr: kdata.addr, len: kdata.num << PAGE_SHIFT);
815	} else {
816	unsigned int domid =
817	(xdata.flags & XENMEM_rsrc_acq_caller_owned) ?
818	DOMID_SELF : kdata.dom;
819	int num, *errs = (int *)pfns;
820
821	BUILD_BUG_ON(sizeof(*errs) > sizeof(*pfns));
822	num = xen_remap_domain_mfn_array(vma,
823	addr: kdata.addr & PAGE_MASK,
824	mfn: pfns, nr: kdata.num, err_ptr: errs,
825	prot: vma->vm_page_prot,
826	domid);
827	if (num < 0)
828	rc = num;
829	else if (num != kdata.num) {
830	unsigned int i;
831
832	for (i = 0; i < num; i++) {
833	rc = errs[i];
834	if (rc < 0)
835	break;
836	}
837	} else
838	rc = 0;
839	}
840
841	out:
842	mmap_write_unlock(mm);
843	kfree(objp: pfns);
844
845	return rc;
846	}
847
848	static long privcmd_ioctl_pcidev_get_gsi(struct file *file, void __user *udata)
849	{
850	#if defined(CONFIG_XEN_ACPI)
851	int rc;
852	struct privcmd_pcidev_get_gsi kdata;
853
854	if (copy_from_user(to: &kdata, from: udata, n: sizeof(kdata)))
855	return -EFAULT;
856
857	rc = xen_acpi_get_gsi_from_sbdf(sbdf: kdata.sbdf);
858	if (rc < 0)
859	return rc;
860
861	kdata.gsi = rc;
862	if (copy_to_user(to: udata, from: &kdata, n: sizeof(kdata)))
863	return -EFAULT;
864
865	return 0;
866	#else
867	return -EINVAL;
868	#endif
869	}
870
871	#ifdef CONFIG_XEN_PRIVCMD_EVENTFD
872	/* Irqfd support */
873	static struct workqueue_struct *irqfd_cleanup_wq;
874	static DEFINE_SPINLOCK(irqfds_lock);
875	DEFINE_STATIC_SRCU(irqfds_srcu);
876	static LIST_HEAD(irqfds_list);
877
878	struct privcmd_kernel_irqfd {
879	struct xen_dm_op_buf xbufs;
880	domid_t dom;
881	bool error;
882	struct eventfd_ctx *eventfd;
883	struct work_struct shutdown;
884	wait_queue_entry_t wait;
885	struct list_head list;
886	poll_table pt;
887	};
888
889	static void irqfd_deactivate(struct privcmd_kernel_irqfd *kirqfd)
890	{
891	lockdep_assert_held(&irqfds_lock);
892
893	list_del_init(entry: &kirqfd->list);
894	queue_work(wq: irqfd_cleanup_wq, work: &kirqfd->shutdown);
895	}
896
897	static void irqfd_shutdown(struct work_struct *work)
898	{
899	struct privcmd_kernel_irqfd *kirqfd =
900	container_of(work, struct privcmd_kernel_irqfd, shutdown);
901	u64 cnt;
902
903	/* Make sure irqfd has been initialized in assign path */
904	synchronize_srcu(ssp: &irqfds_srcu);
905
906	eventfd_ctx_remove_wait_queue(ctx: kirqfd->eventfd, wait: &kirqfd->wait, cnt: &cnt);
907	eventfd_ctx_put(ctx: kirqfd->eventfd);
908	kfree(objp: kirqfd);
909	}
910
911	static void irqfd_inject(struct privcmd_kernel_irqfd *kirqfd)
912	{
913	u64 cnt;
914	long rc;
915
916	eventfd_ctx_do_read(ctx: kirqfd->eventfd, cnt: &cnt);
917
918	xen_preemptible_hcall_begin();
919	rc = HYPERVISOR_dm_op(dom: kirqfd->dom, nr_bufs: 1, bufs: &kirqfd->xbufs);
920	xen_preemptible_hcall_end();
921
922	/* Don't repeat the error message for consecutive failures */
923	if (rc && !kirqfd->error) {
924	pr_err("Failed to configure irq for guest domain: %d\n",
925	kirqfd->dom);
926	}
927
928	kirqfd->error = rc;
929	}
930
931	static int
932	irqfd_wakeup(wait_queue_entry_t *wait, unsigned int mode, int sync, void *key)
933	{
934	struct privcmd_kernel_irqfd *kirqfd =
935	container_of(wait, struct privcmd_kernel_irqfd, wait);
936	__poll_t flags = key_to_poll(key);
937
938	if (flags & EPOLLIN)
939	irqfd_inject(kirqfd);
940
941	if (flags & EPOLLHUP) {
942	unsigned long flags;
943
944	spin_lock_irqsave(&irqfds_lock, flags);
945	irqfd_deactivate(kirqfd);
946	spin_unlock_irqrestore(lock: &irqfds_lock, flags);
947	}
948
949	return 0;
950	}
951
952	static void
953	irqfd_poll_func(struct file *file, wait_queue_head_t *wqh, poll_table *pt)
954	{
955	struct privcmd_kernel_irqfd *kirqfd =
956	container_of(pt, struct privcmd_kernel_irqfd, pt);
957
958	add_wait_queue_priority(wq_head: wqh, wq_entry: &kirqfd->wait);
959	}
960
961	static int privcmd_irqfd_assign(struct privcmd_irqfd *irqfd)
962	{
963	struct privcmd_kernel_irqfd *kirqfd, *tmp;
964	unsigned long flags;
965	__poll_t events;
966	void *dm_op;
967	int ret, idx;
968
969	CLASS(fd, f)(fd: irqfd->fd);
970
971	kirqfd = kzalloc(sizeof(*kirqfd) + irqfd->size, GFP_KERNEL);
972	if (!kirqfd)
973	return -ENOMEM;
974	dm_op = kirqfd + 1;
975
976	if (copy_from_user(to: dm_op, u64_to_user_ptr(irqfd->dm_op), n: irqfd->size)) {
977	ret = -EFAULT;
978	goto error_kfree;
979	}
980
981	kirqfd->xbufs.size = irqfd->size;
982	set_xen_guest_handle(kirqfd->xbufs.h, dm_op);
983	kirqfd->dom = irqfd->dom;
984	INIT_WORK(&kirqfd->shutdown, irqfd_shutdown);
985
986	if (fd_empty(f)) {
987	ret = -EBADF;
988	goto error_kfree;
989	}
990
991	kirqfd->eventfd = eventfd_ctx_fileget(fd_file(f));
992	if (IS_ERR(ptr: kirqfd->eventfd)) {
993	ret = PTR_ERR(ptr: kirqfd->eventfd);
994	goto error_kfree;
995	}
996
997	/*
998	* Install our own custom wake-up handling so we are notified via a
999	* callback whenever someone signals the underlying eventfd.
1000	*/
1001	init_waitqueue_func_entry(wq_entry: &kirqfd->wait, func: irqfd_wakeup);
1002	init_poll_funcptr(pt: &kirqfd->pt, qproc: irqfd_poll_func);
1003
1004	spin_lock_irqsave(&irqfds_lock, flags);
1005
1006	list_for_each_entry(tmp, &irqfds_list, list) {
1007	if (kirqfd->eventfd == tmp->eventfd) {
1008	ret = -EBUSY;
1009	spin_unlock_irqrestore(lock: &irqfds_lock, flags);
1010	goto error_eventfd;
1011	}
1012	}
1013
1014	idx = srcu_read_lock(ssp: &irqfds_srcu);
1015	list_add_tail(new: &kirqfd->list, head: &irqfds_list);
1016	spin_unlock_irqrestore(lock: &irqfds_lock, flags);
1017
1018	/*
1019	* Check if there was an event already pending on the eventfd before we
1020	* registered, and trigger it as if we didn't miss it.
1021	*/
1022	events = vfs_poll(fd_file(f), pt: &kirqfd->pt);
1023	if (events & EPOLLIN)
1024	irqfd_inject(kirqfd);
1025
1026	srcu_read_unlock(ssp: &irqfds_srcu, idx);
1027	return 0;
1028
1029	error_eventfd:
1030	eventfd_ctx_put(ctx: kirqfd->eventfd);
1031
1032	error_kfree:
1033	kfree(objp: kirqfd);
1034	return ret;
1035	}
1036
1037	static int privcmd_irqfd_deassign(struct privcmd_irqfd *irqfd)
1038	{
1039	struct privcmd_kernel_irqfd *kirqfd;
1040	struct eventfd_ctx *eventfd;
1041	unsigned long flags;
1042
1043	eventfd = eventfd_ctx_fdget(fd: irqfd->fd);
1044	if (IS_ERR(ptr: eventfd))
1045	return PTR_ERR(ptr: eventfd);
1046
1047	spin_lock_irqsave(&irqfds_lock, flags);
1048
1049	list_for_each_entry(kirqfd, &irqfds_list, list) {
1050	if (kirqfd->eventfd == eventfd) {
1051	irqfd_deactivate(kirqfd);
1052	break;
1053	}
1054	}
1055
1056	spin_unlock_irqrestore(lock: &irqfds_lock, flags);
1057
1058	eventfd_ctx_put(ctx: eventfd);
1059
1060	/*
1061	* Block until we know all outstanding shutdown jobs have completed so
1062	* that we guarantee there will not be any more interrupts once this
1063	* deassign function returns.
1064	*/
1065	flush_workqueue(irqfd_cleanup_wq);
1066
1067	return 0;
1068	}
1069
1070	static long privcmd_ioctl_irqfd(struct file *file, void __user *udata)
1071	{
1072	struct privcmd_data *data = file->private_data;
1073	struct privcmd_irqfd irqfd;
1074
1075	if (copy_from_user(to: &irqfd, from: udata, n: sizeof(irqfd)))
1076	return -EFAULT;
1077
1078	/* No other flags should be set */
1079	if (irqfd.flags & ~PRIVCMD_IRQFD_FLAG_DEASSIGN)
1080	return -EINVAL;
1081
1082	/* If restriction is in place, check the domid matches */
1083	if (data->domid != DOMID_INVALID && data->domid != irqfd.dom)
1084	return -EPERM;
1085
1086	if (irqfd.flags & PRIVCMD_IRQFD_FLAG_DEASSIGN)
1087	return privcmd_irqfd_deassign(irqfd: &irqfd);
1088
1089	return privcmd_irqfd_assign(irqfd: &irqfd);
1090	}
1091
1092	static int privcmd_irqfd_init(void)
1093	{
1094	irqfd_cleanup_wq = alloc_workqueue("privcmd-irqfd-cleanup", 0, 0);
1095	if (!irqfd_cleanup_wq)
1096	return -ENOMEM;
1097
1098	return 0;
1099	}
1100
1101	static void privcmd_irqfd_exit(void)
1102	{
1103	struct privcmd_kernel_irqfd *kirqfd, *tmp;
1104	unsigned long flags;
1105
1106	spin_lock_irqsave(&irqfds_lock, flags);
1107
1108	list_for_each_entry_safe(kirqfd, tmp, &irqfds_list, list)
1109	irqfd_deactivate(kirqfd);
1110
1111	spin_unlock_irqrestore(lock: &irqfds_lock, flags);
1112
1113	destroy_workqueue(wq: irqfd_cleanup_wq);
1114	}
1115
1116	/* Ioeventfd Support */
1117	#define QUEUE_NOTIFY_VQ_MASK 0xFFFF
1118
1119	static DEFINE_MUTEX(ioreq_lock);
1120	static LIST_HEAD(ioreq_list);
1121
1122	/* per-eventfd structure */
1123	struct privcmd_kernel_ioeventfd {
1124	struct eventfd_ctx *eventfd;
1125	struct list_head list;
1126	u64 addr;
1127	unsigned int addr_len;
1128	unsigned int vq;
1129	};
1130
1131	/* per-guest CPU / port structure */
1132	struct ioreq_port {
1133	int vcpu;
1134	unsigned int port;
1135	struct privcmd_kernel_ioreq *kioreq;
1136	};
1137
1138	/* per-guest structure */
1139	struct privcmd_kernel_ioreq {
1140	domid_t dom;
1141	unsigned int vcpus;
1142	u64 uioreq;
1143	struct ioreq *ioreq;
1144	spinlock_t lock; /* Protects ioeventfds list */
1145	struct list_head ioeventfds;
1146	struct list_head list;
1147	struct ioreq_port ports[] __counted_by(vcpus);
1148	};
1149
1150	static irqreturn_t ioeventfd_interrupt(int irq, void *dev_id)
1151	{
1152	struct ioreq_port *port = dev_id;
1153	struct privcmd_kernel_ioreq *kioreq = port->kioreq;
1154	struct ioreq *ioreq = &kioreq->ioreq[port->vcpu];
1155	struct privcmd_kernel_ioeventfd *kioeventfd;
1156	unsigned int state = STATE_IOREQ_READY;
1157
1158	if (ioreq->state != STATE_IOREQ_READY ||
1159	ioreq->type != IOREQ_TYPE_COPY || ioreq->dir != IOREQ_WRITE)
1160	return IRQ_NONE;
1161
1162	/*
1163	* We need a barrier, smp_mb(), here to ensure reads are finished before
1164	* `state` is updated. Since the lock implementation ensures that
1165	* appropriate barrier will be added anyway, we can avoid adding
1166	* explicit barrier here.
1167	*
1168	* Ideally we don't need to update `state` within the locks, but we do
1169	* that here to avoid adding explicit barrier.
1170	*/
1171
1172	spin_lock(lock: &kioreq->lock);
1173	ioreq->state = STATE_IOREQ_INPROCESS;
1174
1175	list_for_each_entry(kioeventfd, &kioreq->ioeventfds, list) {
1176	if (ioreq->addr == kioeventfd->addr + VIRTIO_MMIO_QUEUE_NOTIFY &&
1177	ioreq->size == kioeventfd->addr_len &&
1178	(ioreq->data & QUEUE_NOTIFY_VQ_MASK) == kioeventfd->vq) {
1179	eventfd_signal(ctx: kioeventfd->eventfd);
1180	state = STATE_IORESP_READY;
1181	break;
1182	}
1183	}
1184	spin_unlock(lock: &kioreq->lock);
1185
1186	/*
1187	* We need a barrier, smp_mb(), here to ensure writes are finished
1188	* before `state` is updated. Since the lock implementation ensures that
1189	* appropriate barrier will be added anyway, we can avoid adding
1190	* explicit barrier here.
1191	*/
1192
1193	ioreq->state = state;
1194
1195	if (state == STATE_IORESP_READY) {
1196	notify_remote_via_evtchn(port: port->port);
1197	return IRQ_HANDLED;
1198	}
1199
1200	return IRQ_NONE;
1201	}
1202
1203	static void ioreq_free(struct privcmd_kernel_ioreq *kioreq)
1204	{
1205	struct ioreq_port *ports = kioreq->ports;
1206	int i;
1207
1208	lockdep_assert_held(&ioreq_lock);
1209
1210	list_del(entry: &kioreq->list);
1211
1212	for (i = kioreq->vcpus - 1; i >= 0; i--)
1213	unbind_from_irqhandler(irq: irq_from_evtchn(evtchn: ports[i].port), dev_id: &ports[i]);
1214
1215	kfree(objp: kioreq);
1216	}
1217
1218	static
1219	struct privcmd_kernel_ioreq *alloc_ioreq(struct privcmd_ioeventfd *ioeventfd)
1220	{
1221	struct privcmd_kernel_ioreq *kioreq;
1222	struct mm_struct *mm = current->mm;
1223	struct vm_area_struct *vma;
1224	struct page **pages;
1225	unsigned int *ports;
1226	int ret, size, i;
1227
1228	lockdep_assert_held(&ioreq_lock);
1229
1230	size = struct_size(kioreq, ports, ioeventfd->vcpus);
1231	kioreq = kzalloc(size, GFP_KERNEL);
1232	if (!kioreq)
1233	return ERR_PTR(error: -ENOMEM);
1234
1235	kioreq->dom = ioeventfd->dom;
1236	kioreq->vcpus = ioeventfd->vcpus;
1237	kioreq->uioreq = ioeventfd->ioreq;
1238	spin_lock_init(&kioreq->lock);
1239	INIT_LIST_HEAD(list: &kioreq->ioeventfds);
1240
1241	/* The memory for ioreq server must have been mapped earlier */
1242	mmap_write_lock(mm);
1243	vma = find_vma(mm, addr: (unsigned long)ioeventfd->ioreq);
1244	if (!vma) {
1245	pr_err("Failed to find vma for ioreq page!\n");
1246	mmap_write_unlock(mm);
1247	ret = -EFAULT;
1248	goto error_kfree;
1249	}
1250
1251	pages = vma->vm_private_data;
1252	kioreq->ioreq = (struct ioreq *)(page_to_virt(pages[0]));
1253	mmap_write_unlock(mm);
1254
1255	ports = memdup_array_user(u64_to_user_ptr(ioeventfd->ports),
1256	n: kioreq->vcpus, size: sizeof(*ports));
1257	if (IS_ERR(ptr: ports)) {
1258	ret = PTR_ERR(ptr: ports);
1259	goto error_kfree;
1260	}
1261
1262	for (i = 0; i < kioreq->vcpus; i++) {
1263	kioreq->ports[i].vcpu = i;
1264	kioreq->ports[i].port = ports[i];
1265	kioreq->ports[i].kioreq = kioreq;
1266
1267	ret = bind_evtchn_to_irqhandler_lateeoi(evtchn: ports[i],
1268	handler: ioeventfd_interrupt, IRQF_SHARED, devname: "ioeventfd",
1269	dev_id: &kioreq->ports[i]);
1270	if (ret < 0)
1271	goto error_unbind;
1272	}
1273
1274	kfree(objp: ports);
1275
1276	list_add_tail(new: &kioreq->list, head: &ioreq_list);
1277
1278	return kioreq;
1279
1280	error_unbind:
1281	while (--i >= 0)
1282	unbind_from_irqhandler(irq: irq_from_evtchn(evtchn: ports[i]), dev_id: &kioreq->ports[i]);
1283
1284	kfree(objp: ports);
1285	error_kfree:
1286	kfree(objp: kioreq);
1287	return ERR_PTR(error: ret);
1288	}
1289
1290	static struct privcmd_kernel_ioreq *
1291	get_ioreq(struct privcmd_ioeventfd *ioeventfd, struct eventfd_ctx *eventfd)
1292	{
1293	struct privcmd_kernel_ioreq *kioreq;
1294	unsigned long flags;
1295
1296	list_for_each_entry(kioreq, &ioreq_list, list) {
1297	struct privcmd_kernel_ioeventfd *kioeventfd;
1298
1299	/*
1300	* kioreq fields can be accessed here without a lock as they are
1301	* never updated after being added to the ioreq_list.
1302	*/
1303	if (kioreq->uioreq != ioeventfd->ioreq) {
1304	continue;
1305	} else if (kioreq->dom != ioeventfd->dom ||
1306	kioreq->vcpus != ioeventfd->vcpus) {
1307	pr_err("Invalid ioeventfd configuration mismatch, dom (%u vs %u), vcpus (%u vs %u)\n",
1308	kioreq->dom, ioeventfd->dom, kioreq->vcpus,
1309	ioeventfd->vcpus);
1310	return ERR_PTR(error: -EINVAL);
1311	}
1312
1313	/* Look for a duplicate eventfd for the same guest */
1314	spin_lock_irqsave(&kioreq->lock, flags);
1315	list_for_each_entry(kioeventfd, &kioreq->ioeventfds, list) {
1316	if (eventfd == kioeventfd->eventfd) {
1317	spin_unlock_irqrestore(lock: &kioreq->lock, flags);
1318	return ERR_PTR(error: -EBUSY);
1319	}
1320	}
1321	spin_unlock_irqrestore(lock: &kioreq->lock, flags);
1322
1323	return kioreq;
1324	}
1325
1326	/* Matching kioreq isn't found, allocate a new one */
1327	return alloc_ioreq(ioeventfd);
1328	}
1329
1330	static void ioeventfd_free(struct privcmd_kernel_ioeventfd *kioeventfd)
1331	{
1332	list_del(entry: &kioeventfd->list);
1333	eventfd_ctx_put(ctx: kioeventfd->eventfd);
1334	kfree(objp: kioeventfd);
1335	}
1336
1337	static int privcmd_ioeventfd_assign(struct privcmd_ioeventfd *ioeventfd)
1338	{
1339	struct privcmd_kernel_ioeventfd *kioeventfd;
1340	struct privcmd_kernel_ioreq *kioreq;
1341	unsigned long flags;
1342	int ret;
1343
1344	/* Check for range overflow */
1345	if (ioeventfd->addr + ioeventfd->addr_len < ioeventfd->addr)
1346	return -EINVAL;
1347
1348	/* Vhost requires us to support length 1, 2, 4, and 8 */
1349	if (!(ioeventfd->addr_len == 1 || ioeventfd->addr_len == 2 ||
1350	ioeventfd->addr_len == 4 || ioeventfd->addr_len == 8))
1351	return -EINVAL;
1352
1353	/* 4096 vcpus limit enough ? */
1354	if (!ioeventfd->vcpus || ioeventfd->vcpus > 4096)
1355	return -EINVAL;
1356
1357	kioeventfd = kzalloc(sizeof(*kioeventfd), GFP_KERNEL);
1358	if (!kioeventfd)
1359	return -ENOMEM;
1360
1361	kioeventfd->eventfd = eventfd_ctx_fdget(fd: ioeventfd->event_fd);
1362	if (IS_ERR(ptr: kioeventfd->eventfd)) {
1363	ret = PTR_ERR(ptr: kioeventfd->eventfd);
1364	goto error_kfree;
1365	}
1366
1367	kioeventfd->addr = ioeventfd->addr;
1368	kioeventfd->addr_len = ioeventfd->addr_len;
1369	kioeventfd->vq = ioeventfd->vq;
1370
1371	mutex_lock(&ioreq_lock);
1372	kioreq = get_ioreq(ioeventfd, eventfd: kioeventfd->eventfd);
1373	if (IS_ERR(ptr: kioreq)) {
1374	mutex_unlock(lock: &ioreq_lock);
1375	ret = PTR_ERR(ptr: kioreq);
1376	goto error_eventfd;
1377	}
1378
1379	spin_lock_irqsave(&kioreq->lock, flags);
1380	list_add_tail(new: &kioeventfd->list, head: &kioreq->ioeventfds);
1381	spin_unlock_irqrestore(lock: &kioreq->lock, flags);
1382
1383	mutex_unlock(lock: &ioreq_lock);
1384
1385	return 0;
1386
1387	error_eventfd:
1388	eventfd_ctx_put(ctx: kioeventfd->eventfd);
1389
1390	error_kfree:
1391	kfree(objp: kioeventfd);
1392	return ret;
1393	}
1394
1395	static int privcmd_ioeventfd_deassign(struct privcmd_ioeventfd *ioeventfd)
1396	{
1397	struct privcmd_kernel_ioreq *kioreq, *tkioreq;
1398	struct eventfd_ctx *eventfd;
1399	unsigned long flags;
1400	int ret = 0;
1401
1402	eventfd = eventfd_ctx_fdget(fd: ioeventfd->event_fd);
1403	if (IS_ERR(ptr: eventfd))
1404	return PTR_ERR(ptr: eventfd);
1405
1406	mutex_lock(&ioreq_lock);
1407	list_for_each_entry_safe(kioreq, tkioreq, &ioreq_list, list) {
1408	struct privcmd_kernel_ioeventfd *kioeventfd, *tmp;
1409	/*
1410	* kioreq fields can be accessed here without a lock as they are
1411	* never updated after being added to the ioreq_list.
1412	*/
1413	if (kioreq->dom != ioeventfd->dom ||
1414	kioreq->uioreq != ioeventfd->ioreq ||
1415	kioreq->vcpus != ioeventfd->vcpus)
1416	continue;
1417
1418	spin_lock_irqsave(&kioreq->lock, flags);
1419	list_for_each_entry_safe(kioeventfd, tmp, &kioreq->ioeventfds, list) {
1420	if (eventfd == kioeventfd->eventfd) {
1421	ioeventfd_free(kioeventfd);
1422	spin_unlock_irqrestore(lock: &kioreq->lock, flags);
1423
1424	if (list_empty(head: &kioreq->ioeventfds))
1425	ioreq_free(kioreq);
1426	goto unlock;
1427	}
1428	}
1429	spin_unlock_irqrestore(lock: &kioreq->lock, flags);
1430	break;
1431	}
1432
1433	pr_err("Ioeventfd isn't already assigned, dom: %u, addr: %llu\n",
1434	ioeventfd->dom, ioeventfd->addr);
1435	ret = -ENODEV;
1436
1437	unlock:
1438	mutex_unlock(lock: &ioreq_lock);
1439	eventfd_ctx_put(ctx: eventfd);
1440
1441	return ret;
1442	}
1443
1444	static long privcmd_ioctl_ioeventfd(struct file *file, void __user *udata)
1445	{
1446	struct privcmd_data *data = file->private_data;
1447	struct privcmd_ioeventfd ioeventfd;
1448
1449	if (copy_from_user(to: &ioeventfd, from: udata, n: sizeof(ioeventfd)))
1450	return -EFAULT;
1451
1452	/* No other flags should be set */
1453	if (ioeventfd.flags & ~PRIVCMD_IOEVENTFD_FLAG_DEASSIGN)
1454	return -EINVAL;
1455
1456	/* If restriction is in place, check the domid matches */
1457	if (data->domid != DOMID_INVALID && data->domid != ioeventfd.dom)
1458	return -EPERM;
1459
1460	if (ioeventfd.flags & PRIVCMD_IOEVENTFD_FLAG_DEASSIGN)
1461	return privcmd_ioeventfd_deassign(ioeventfd: &ioeventfd);
1462
1463	return privcmd_ioeventfd_assign(ioeventfd: &ioeventfd);
1464	}
1465
1466	static void privcmd_ioeventfd_exit(void)
1467	{
1468	struct privcmd_kernel_ioreq *kioreq, *tmp;
1469	unsigned long flags;
1470
1471	mutex_lock(&ioreq_lock);
1472	list_for_each_entry_safe(kioreq, tmp, &ioreq_list, list) {
1473	struct privcmd_kernel_ioeventfd *kioeventfd, *tmp;
1474
1475	spin_lock_irqsave(&kioreq->lock, flags);
1476	list_for_each_entry_safe(kioeventfd, tmp, &kioreq->ioeventfds, list)
1477	ioeventfd_free(kioeventfd);
1478	spin_unlock_irqrestore(lock: &kioreq->lock, flags);
1479
1480	ioreq_free(kioreq);
1481	}
1482	mutex_unlock(lock: &ioreq_lock);
1483	}
1484	#else
1485	static inline long privcmd_ioctl_irqfd(struct file *file, void __user *udata)
1486	{
1487	return -EOPNOTSUPP;
1488	}
1489
1490	static inline int privcmd_irqfd_init(void)
1491	{
1492	return 0;
1493	}
1494
1495	static inline void privcmd_irqfd_exit(void)
1496	{
1497	}
1498
1499	static inline long privcmd_ioctl_ioeventfd(struct file *file, void __user *udata)
1500	{
1501	return -EOPNOTSUPP;
1502	}
1503
1504	static inline void privcmd_ioeventfd_exit(void)
1505	{
1506	}
1507	#endif /* CONFIG_XEN_PRIVCMD_EVENTFD */
1508
1509	static long privcmd_ioctl(struct file *file,
1510	unsigned int cmd, unsigned long data)
1511	{
1512	int ret = -ENOTTY;
1513	void __user *udata = (void __user *) data;
1514
1515	switch (cmd) {
1516	case IOCTL_PRIVCMD_HYPERCALL:
1517	ret = privcmd_ioctl_hypercall(file, udata);
1518	break;
1519
1520	case IOCTL_PRIVCMD_MMAP:
1521	ret = privcmd_ioctl_mmap(file, udata);
1522	break;
1523
1524	case IOCTL_PRIVCMD_MMAPBATCH:
1525	ret = privcmd_ioctl_mmap_batch(file, udata, version: 1);
1526	break;
1527
1528	case IOCTL_PRIVCMD_MMAPBATCH_V2:
1529	ret = privcmd_ioctl_mmap_batch(file, udata, version: 2);
1530	break;
1531
1532	case IOCTL_PRIVCMD_DM_OP:
1533	ret = privcmd_ioctl_dm_op(file, udata);
1534	break;
1535
1536	case IOCTL_PRIVCMD_RESTRICT:
1537	ret = privcmd_ioctl_restrict(file, udata);
1538	break;
1539
1540	case IOCTL_PRIVCMD_MMAP_RESOURCE:
1541	ret = privcmd_ioctl_mmap_resource(file, udata);
1542	break;
1543
1544	case IOCTL_PRIVCMD_IRQFD:
1545	ret = privcmd_ioctl_irqfd(file, udata);
1546	break;
1547
1548	case IOCTL_PRIVCMD_IOEVENTFD:
1549	ret = privcmd_ioctl_ioeventfd(file, udata);
1550	break;
1551
1552	case IOCTL_PRIVCMD_PCIDEV_GET_GSI:
1553	ret = privcmd_ioctl_pcidev_get_gsi(file, udata);
1554	break;
1555
1556	default:
1557	break;
1558	}
1559
1560	return ret;
1561	}
1562
1563	static int privcmd_open(struct inode *ino, struct file *file)
1564	{
1565	struct privcmd_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
1566
1567	if (!data)
1568	return -ENOMEM;
1569
1570	/* DOMID_INVALID implies no restriction */
1571	data->domid = DOMID_INVALID;
1572
1573	file->private_data = data;
1574	return 0;
1575	}
1576
1577	static int privcmd_release(struct inode *ino, struct file *file)
1578	{
1579	struct privcmd_data *data = file->private_data;
1580
1581	kfree(objp: data);
1582	return 0;
1583	}
1584
1585	static void privcmd_close(struct vm_area_struct *vma)
1586	{
1587	struct page **pages = vma->vm_private_data;
1588	int numpgs = vma_pages(vma);
1589	int numgfns = (vma->vm_end - vma->vm_start) >> XEN_PAGE_SHIFT;
1590	int rc;
1591
1592	if (xen_pv_domain() || !numpgs || !pages)
1593	return;
1594
1595	rc = xen_unmap_domain_gfn_range(vma, numpgs: numgfns, pages);
1596	if (rc == 0)
1597	xen_free_unpopulated_pages(nr_pages: numpgs, pages);
1598	else
1599	pr_crit("unable to unmap MFN range: leaking %d pages. rc=%d\n",
1600	numpgs, rc);
1601	kvfree(addr: pages);
1602	}
1603
1604	static vm_fault_t privcmd_fault(struct vm_fault *vmf)
1605	{
1606	printk(KERN_DEBUG "privcmd_fault: vma=%p %lx-%lx, pgoff=%lx, uv=%p\n",
1607	vmf->vma, vmf->vma->vm_start, vmf->vma->vm_end,
1608	vmf->pgoff, (void *)vmf->address);
1609
1610	return VM_FAULT_SIGBUS;
1611	}
1612
1613	static const struct vm_operations_struct privcmd_vm_ops = {
1614	.close = privcmd_close,
1615	.fault = privcmd_fault
1616	};
1617
1618	static int privcmd_mmap(struct file *file, struct vm_area_struct *vma)
1619	{
1620	/* DONTCOPY is essential for Xen because copy_page_range doesn't know
1621	* how to recreate these mappings */
1622	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTCOPY |
1623	VM_DONTEXPAND | VM_DONTDUMP);
1624	vma->vm_ops = &privcmd_vm_ops;
1625	vma->vm_private_data = NULL;
1626
1627	return 0;
1628	}
1629
1630	/*
1631	* For MMAPBATCH*. This allows asserting the singleshot mapping
1632	* on a per pfn/pte basis. Mapping calls that fail with ENOENT
1633	* can be then retried until success.
1634	*/
1635	static int is_mapped_fn(pte_t *pte, unsigned long addr, void *data)
1636	{
1637	return pte_none(pte: ptep_get(ptep: pte)) ? 0 : -EBUSY;
1638	}
1639
1640	static int privcmd_vma_range_is_mapped(
1641	struct vm_area_struct *vma,
1642	unsigned long addr,
1643	unsigned long nr_pages)
1644	{
1645	return apply_to_page_range(mm: vma->vm_mm, address: addr, size: nr_pages << PAGE_SHIFT,
1646	fn: is_mapped_fn, NULL) != 0;
1647	}
1648
1649	const struct file_operations xen_privcmd_fops = {
1650	.owner = THIS_MODULE,
1651	.unlocked_ioctl = privcmd_ioctl,
1652	.open = privcmd_open,
1653	.release = privcmd_release,
1654	.mmap = privcmd_mmap,
1655	};
1656	EXPORT_SYMBOL_GPL(xen_privcmd_fops);
1657
1658	static struct miscdevice privcmd_dev = {
1659	.minor = MISC_DYNAMIC_MINOR,
1660	.name = "xen/privcmd",
1661	.fops = &xen_privcmd_fops,
1662	};
1663
1664	static int __init privcmd_init(void)
1665	{
1666	int err;
1667
1668	if (!xen_domain())
1669	return -ENODEV;
1670
1671	err = misc_register(misc: &privcmd_dev);
1672	if (err != 0) {
1673	pr_err("Could not register Xen privcmd device\n");
1674	return err;
1675	}
1676
1677	err = misc_register(misc: &xen_privcmdbuf_dev);
1678	if (err != 0) {
1679	pr_err("Could not register Xen hypercall-buf device\n");
1680	goto err_privcmdbuf;
1681	}
1682
1683	err = privcmd_irqfd_init();
1684	if (err != 0) {
1685	pr_err("irqfd init failed\n");
1686	goto err_irqfd;
1687	}
1688
1689	return 0;
1690
1691	err_irqfd:
1692	misc_deregister(misc: &xen_privcmdbuf_dev);
1693	err_privcmdbuf:
1694	misc_deregister(misc: &privcmd_dev);
1695	return err;
1696	}
1697
1698	static void __exit privcmd_exit(void)
1699	{
1700	privcmd_ioeventfd_exit();
1701	privcmd_irqfd_exit();
1702	misc_deregister(misc: &privcmd_dev);
1703	misc_deregister(misc: &xen_privcmdbuf_dev);
1704	}
1705
1706	module_init(privcmd_init);
1707	module_exit(privcmd_exit);
1708