1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2024 Loongson Technology Corporation Limited
4 */
5
6#include <linux/kvm_host.h>
7#include <asm/kvm_ipi.h>
8#include <asm/kvm_vcpu.h>
9
10static void ipi_set(struct kvm_vcpu *vcpu, uint32_t data)
11{
12 uint32_t status;
13 struct kvm_interrupt irq;
14
15 spin_lock(lock: &vcpu->arch.ipi_state.lock);
16 status = vcpu->arch.ipi_state.status;
17 vcpu->arch.ipi_state.status |= data;
18 spin_unlock(lock: &vcpu->arch.ipi_state.lock);
19 if ((status == 0) && data) {
20 irq.irq = LARCH_INT_IPI;
21 kvm_vcpu_ioctl_interrupt(vcpu, &irq);
22 }
23}
24
25static void ipi_send(struct kvm *kvm, uint64_t data)
26{
27 int cpu;
28 struct kvm_vcpu *vcpu;
29
30 cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
31 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
32 if (unlikely(vcpu == NULL)) {
33 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
34 return;
35 }
36
37 ipi_set(vcpu, BIT(data & 0x1f));
38}
39
40static void ipi_clear(struct kvm_vcpu *vcpu, uint64_t data)
41{
42 uint32_t status;
43 struct kvm_interrupt irq;
44
45 spin_lock(lock: &vcpu->arch.ipi_state.lock);
46 vcpu->arch.ipi_state.status &= ~data;
47 status = vcpu->arch.ipi_state.status;
48 spin_unlock(lock: &vcpu->arch.ipi_state.lock);
49 if (status == 0) {
50 irq.irq = -LARCH_INT_IPI;
51 kvm_vcpu_ioctl_interrupt(vcpu, &irq);
52 }
53}
54
55static uint64_t read_mailbox(struct kvm_vcpu *vcpu, int offset, int len)
56{
57 uint64_t data = 0;
58
59 spin_lock(lock: &vcpu->arch.ipi_state.lock);
60 data = *(ulong *)((void *)vcpu->arch.ipi_state.buf + (offset - 0x20));
61 spin_unlock(lock: &vcpu->arch.ipi_state.lock);
62
63 switch (len) {
64 case 1:
65 return data & 0xff;
66 case 2:
67 return data & 0xffff;
68 case 4:
69 return data & 0xffffffff;
70 case 8:
71 return data;
72 default:
73 kvm_err("%s: unknown data len: %d\n", __func__, len);
74 return 0;
75 }
76}
77
78static void write_mailbox(struct kvm_vcpu *vcpu, int offset, uint64_t data, int len)
79{
80 void *pbuf;
81
82 spin_lock(lock: &vcpu->arch.ipi_state.lock);
83 pbuf = (void *)vcpu->arch.ipi_state.buf + (offset - 0x20);
84
85 switch (len) {
86 case 1:
87 *(unsigned char *)pbuf = (unsigned char)data;
88 break;
89 case 2:
90 *(unsigned short *)pbuf = (unsigned short)data;
91 break;
92 case 4:
93 *(unsigned int *)pbuf = (unsigned int)data;
94 break;
95 case 8:
96 *(unsigned long *)pbuf = (unsigned long)data;
97 break;
98 default:
99 kvm_err("%s: unknown data len: %d\n", __func__, len);
100 }
101 spin_unlock(lock: &vcpu->arch.ipi_state.lock);
102}
103
104static int mail_send(struct kvm *kvm, uint64_t data)
105{
106 int i, cpu, mailbox, offset;
107 uint32_t val = 0, mask = 0;
108 struct kvm_vcpu *vcpu;
109
110 cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
111 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
112 if (unlikely(vcpu == NULL)) {
113 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
114 return -EINVAL;
115 }
116 mailbox = ((data & 0xffffffff) >> 2) & 0x7;
117 offset = IOCSR_IPI_BUF_20 + mailbox * 4;
118 if ((data >> 27) & 0xf) {
119 val = read_mailbox(vcpu, offset, len: 4);
120 for (i = 0; i < 4; i++)
121 if (data & (BIT(27 + i)))
122 mask |= (0xff << (i * 8));
123 val &= mask;
124 }
125
126 val |= ((uint32_t)(data >> 32) & ~mask);
127 write_mailbox(vcpu, offset, data: val, len: 4);
128
129 return 0;
130}
131
132static int send_ipi_data(struct kvm_vcpu *vcpu, gpa_t addr, uint64_t data)
133{
134 int i, idx, ret;
135 uint64_t val = 0, mask = 0;
136
137 /*
138 * Bit 27-30 is mask for byte writing.
139 * If the mask is 0, we need not to do anything.
140 */
141 if ((data >> 27) & 0xf) {
142 /* Read the old val */
143 idx = srcu_read_lock(ssp: &vcpu->kvm->srcu);
144 ret = kvm_io_bus_read(vcpu, bus_idx: KVM_IOCSR_BUS, addr, len: 4, val: &val);
145 srcu_read_unlock(ssp: &vcpu->kvm->srcu, idx);
146 if (unlikely(ret)) {
147 kvm_err("%s: : read data from addr %llx failed\n", __func__, addr);
148 return ret;
149 }
150 /* Construct the mask by scanning the bit 27-30 */
151 for (i = 0; i < 4; i++) {
152 if (data & (BIT(27 + i)))
153 mask |= (0xff << (i * 8));
154 }
155 /* Save the old part of val */
156 val &= mask;
157 }
158 val |= ((uint32_t)(data >> 32) & ~mask);
159 idx = srcu_read_lock(ssp: &vcpu->kvm->srcu);
160 ret = kvm_io_bus_write(vcpu, bus_idx: KVM_IOCSR_BUS, addr, len: 4, val: &val);
161 srcu_read_unlock(ssp: &vcpu->kvm->srcu, idx);
162 if (unlikely(ret))
163 kvm_err("%s: : write data to addr %llx failed\n", __func__, addr);
164
165 return ret;
166}
167
168static int any_send(struct kvm *kvm, uint64_t data)
169{
170 int cpu, offset;
171 struct kvm_vcpu *vcpu;
172
173 cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
174 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
175 if (unlikely(vcpu == NULL)) {
176 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
177 return -EINVAL;
178 }
179 offset = data & 0xffff;
180
181 return send_ipi_data(vcpu, addr: offset, data);
182}
183
184static int loongarch_ipi_readl(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *val)
185{
186 int ret = 0;
187 uint32_t offset;
188 uint64_t res = 0;
189
190 offset = (uint32_t)(addr & 0x1ff);
191 WARN_ON_ONCE(offset & (len - 1));
192
193 switch (offset) {
194 case IOCSR_IPI_STATUS:
195 spin_lock(lock: &vcpu->arch.ipi_state.lock);
196 res = vcpu->arch.ipi_state.status;
197 spin_unlock(lock: &vcpu->arch.ipi_state.lock);
198 break;
199 case IOCSR_IPI_EN:
200 spin_lock(lock: &vcpu->arch.ipi_state.lock);
201 res = vcpu->arch.ipi_state.en;
202 spin_unlock(lock: &vcpu->arch.ipi_state.lock);
203 break;
204 case IOCSR_IPI_SET:
205 res = 0;
206 break;
207 case IOCSR_IPI_CLEAR:
208 res = 0;
209 break;
210 case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7:
211 if (offset + len > IOCSR_IPI_BUF_38 + 8) {
212 kvm_err("%s: invalid offset or len: offset = %d, len = %d\n",
213 __func__, offset, len);
214 ret = -EINVAL;
215 break;
216 }
217 res = read_mailbox(vcpu, offset, len);
218 break;
219 default:
220 kvm_err("%s: unknown addr: %llx\n", __func__, addr);
221 ret = -EINVAL;
222 break;
223 }
224 *(uint64_t *)val = res;
225
226 return ret;
227}
228
229static int loongarch_ipi_writel(struct kvm_vcpu *vcpu, gpa_t addr, int len, const void *val)
230{
231 int ret = 0;
232 uint64_t data;
233 uint32_t offset;
234
235 data = *(uint64_t *)val;
236
237 offset = (uint32_t)(addr & 0x1ff);
238 WARN_ON_ONCE(offset & (len - 1));
239
240 switch (offset) {
241 case IOCSR_IPI_STATUS:
242 ret = -EINVAL;
243 break;
244 case IOCSR_IPI_EN:
245 spin_lock(lock: &vcpu->arch.ipi_state.lock);
246 vcpu->arch.ipi_state.en = data;
247 spin_unlock(lock: &vcpu->arch.ipi_state.lock);
248 break;
249 case IOCSR_IPI_SET:
250 ipi_set(vcpu, data);
251 break;
252 case IOCSR_IPI_CLEAR:
253 /* Just clear the status of the current vcpu */
254 ipi_clear(vcpu, data);
255 break;
256 case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7:
257 if (offset + len > IOCSR_IPI_BUF_38 + 8) {
258 kvm_err("%s: invalid offset or len: offset = %d, len = %d\n",
259 __func__, offset, len);
260 ret = -EINVAL;
261 break;
262 }
263 write_mailbox(vcpu, offset, data, len);
264 break;
265 case IOCSR_IPI_SEND:
266 ipi_send(kvm: vcpu->kvm, data);
267 break;
268 case IOCSR_MAIL_SEND:
269 ret = mail_send(kvm: vcpu->kvm, data);
270 break;
271 case IOCSR_ANY_SEND:
272 ret = any_send(kvm: vcpu->kvm, data);
273 break;
274 default:
275 kvm_err("%s: unknown addr: %llx\n", __func__, addr);
276 ret = -EINVAL;
277 break;
278 }
279
280 return ret;
281}
282
283static int kvm_ipi_read(struct kvm_vcpu *vcpu,
284 struct kvm_io_device *dev,
285 gpa_t addr, int len, void *val)
286{
287 vcpu->stat.ipi_read_exits++;
288 return loongarch_ipi_readl(vcpu, addr, len, val);
289}
290
291static int kvm_ipi_write(struct kvm_vcpu *vcpu,
292 struct kvm_io_device *dev,
293 gpa_t addr, int len, const void *val)
294{
295 vcpu->stat.ipi_write_exits++;
296 return loongarch_ipi_writel(vcpu, addr, len, val);
297}
298
299static const struct kvm_io_device_ops kvm_ipi_ops = {
300 .read = kvm_ipi_read,
301 .write = kvm_ipi_write,
302};
303
304static int kvm_ipi_regs_access(struct kvm_device *dev,
305 struct kvm_device_attr *attr,
306 bool is_write)
307{
308 int len = 4;
309 int cpu, addr;
310 uint64_t val;
311 void *p = NULL;
312 struct kvm_vcpu *vcpu;
313
314 cpu = (attr->attr >> 16) & 0x3ff;
315 addr = attr->attr & 0xff;
316
317 vcpu = kvm_get_vcpu_by_id(kvm: dev->kvm, id: cpu);
318 if (unlikely(vcpu == NULL)) {
319 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
320 return -EINVAL;
321 }
322
323 switch (addr) {
324 case IOCSR_IPI_STATUS:
325 p = &vcpu->arch.ipi_state.status;
326 break;
327 case IOCSR_IPI_EN:
328 p = &vcpu->arch.ipi_state.en;
329 break;
330 case IOCSR_IPI_SET:
331 p = &vcpu->arch.ipi_state.set;
332 break;
333 case IOCSR_IPI_CLEAR:
334 p = &vcpu->arch.ipi_state.clear;
335 break;
336 case IOCSR_IPI_BUF_20:
337 p = &vcpu->arch.ipi_state.buf[0];
338 len = 8;
339 break;
340 case IOCSR_IPI_BUF_28:
341 p = &vcpu->arch.ipi_state.buf[1];
342 len = 8;
343 break;
344 case IOCSR_IPI_BUF_30:
345 p = &vcpu->arch.ipi_state.buf[2];
346 len = 8;
347 break;
348 case IOCSR_IPI_BUF_38:
349 p = &vcpu->arch.ipi_state.buf[3];
350 len = 8;
351 break;
352 default:
353 kvm_err("%s: unknown ipi register, addr = %d\n", __func__, addr);
354 return -EINVAL;
355 }
356
357 if (is_write) {
358 if (len == 4) {
359 if (get_user(val, (uint32_t __user *)attr->addr))
360 return -EFAULT;
361 *(uint32_t *)p = (uint32_t)val;
362 } else if (len == 8) {
363 if (get_user(val, (uint64_t __user *)attr->addr))
364 return -EFAULT;
365 *(uint64_t *)p = val;
366 }
367 } else {
368 if (len == 4) {
369 val = *(uint32_t *)p;
370 return put_user(val, (uint32_t __user *)attr->addr);
371 } else if (len == 8) {
372 val = *(uint64_t *)p;
373 return put_user(val, (uint64_t __user *)attr->addr);
374 }
375 }
376
377 return 0;
378}
379
380static int kvm_ipi_get_attr(struct kvm_device *dev,
381 struct kvm_device_attr *attr)
382{
383 switch (attr->group) {
384 case KVM_DEV_LOONGARCH_IPI_GRP_REGS:
385 return kvm_ipi_regs_access(dev, attr, is_write: false);
386 default:
387 kvm_err("%s: unknown group (%d)\n", __func__, attr->group);
388 return -EINVAL;
389 }
390}
391
392static int kvm_ipi_set_attr(struct kvm_device *dev,
393 struct kvm_device_attr *attr)
394{
395 switch (attr->group) {
396 case KVM_DEV_LOONGARCH_IPI_GRP_REGS:
397 return kvm_ipi_regs_access(dev, attr, is_write: true);
398 default:
399 kvm_err("%s: unknown group (%d)\n", __func__, attr->group);
400 return -EINVAL;
401 }
402}
403
404static int kvm_ipi_create(struct kvm_device *dev, u32 type)
405{
406 int ret;
407 struct kvm *kvm;
408 struct kvm_io_device *device;
409 struct loongarch_ipi *s;
410
411 if (!dev) {
412 kvm_err("%s: kvm_device ptr is invalid!\n", __func__);
413 return -EINVAL;
414 }
415
416 kvm = dev->kvm;
417 if (kvm->arch.ipi) {
418 kvm_err("%s: LoongArch IPI has already been created!\n", __func__);
419 return -EINVAL;
420 }
421
422 s = kzalloc(sizeof(struct loongarch_ipi), GFP_KERNEL);
423 if (!s)
424 return -ENOMEM;
425
426 spin_lock_init(&s->lock);
427 s->kvm = kvm;
428
429 /*
430 * Initialize IOCSR device
431 */
432 device = &s->device;
433 kvm_iodevice_init(device, &kvm_ipi_ops);
434 mutex_lock(&kvm->slots_lock);
435 ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, IOCSR_IPI_BASE, IOCSR_IPI_SIZE, device);
436 mutex_unlock(lock: &kvm->slots_lock);
437 if (ret < 0) {
438 kvm_err("%s: Initialize IOCSR dev failed, ret = %d\n", __func__, ret);
439 goto err;
440 }
441
442 kvm->arch.ipi = s;
443 return 0;
444
445err:
446 kfree(objp: s);
447 return -EFAULT;
448}
449
450static void kvm_ipi_destroy(struct kvm_device *dev)
451{
452 struct kvm *kvm;
453 struct loongarch_ipi *ipi;
454
455 if (!dev || !dev->kvm || !dev->kvm->arch.ipi)
456 return;
457
458 kvm = dev->kvm;
459 ipi = kvm->arch.ipi;
460 kvm_io_bus_unregister_dev(kvm, bus_idx: KVM_IOCSR_BUS, dev: &ipi->device);
461 kfree(objp: ipi);
462 kfree(objp: dev);
463}
464
465static struct kvm_device_ops kvm_ipi_dev_ops = {
466 .name = "kvm-loongarch-ipi",
467 .create = kvm_ipi_create,
468 .destroy = kvm_ipi_destroy,
469 .set_attr = kvm_ipi_set_attr,
470 .get_attr = kvm_ipi_get_attr,
471};
472
473int kvm_loongarch_register_ipi_device(void)
474{
475 return kvm_register_device_ops(ops: &kvm_ipi_dev_ops, KVM_DEV_TYPE_LOONGARCH_IPI);
476}
477

source code of linux/arch/loongarch/kvm/intc/ipi.c