1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 *
5 * Derived from MIPS:
6 * Copyright (C) 2000, 2001 Kanoj Sarcar
7 * Copyright (C) 2000, 2001 Ralf Baechle
8 * Copyright (C) 2000, 2001 Silicon Graphics, Inc.
9 * Copyright (C) 2000, 2001, 2003 Broadcom Corporation
10 */
11#include <linux/acpi.h>
12#include <linux/cpu.h>
13#include <linux/cpumask.h>
14#include <linux/init.h>
15#include <linux/interrupt.h>
16#include <linux/irq_work.h>
17#include <linux/profile.h>
18#include <linux/seq_file.h>
19#include <linux/smp.h>
20#include <linux/threads.h>
21#include <linux/export.h>
22#include <linux/suspend.h>
23#include <linux/syscore_ops.h>
24#include <linux/time.h>
25#include <linux/tracepoint.h>
26#include <linux/sched/hotplug.h>
27#include <linux/sched/task_stack.h>
28
29#include <asm/cpu.h>
30#include <asm/idle.h>
31#include <asm/loongson.h>
32#include <asm/mmu_context.h>
33#include <asm/numa.h>
34#include <asm/paravirt.h>
35#include <asm/processor.h>
36#include <asm/setup.h>
37#include <asm/time.h>
38
39int __cpu_number_map[NR_CPUS]; /* Map physical to logical */
40EXPORT_SYMBOL(__cpu_number_map);
41
42int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */
43EXPORT_SYMBOL(__cpu_logical_map);
44
45/* Representing the threads (siblings) of each logical CPU */
46cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly;
47EXPORT_SYMBOL(cpu_sibling_map);
48
49/* Representing the last level cache shared map of each logical CPU */
50cpumask_t cpu_llc_shared_map[NR_CPUS] __read_mostly;
51EXPORT_SYMBOL(cpu_llc_shared_map);
52
53/* Representing the core map of multi-core chips of each logical CPU */
54cpumask_t cpu_core_map[NR_CPUS] __read_mostly;
55EXPORT_SYMBOL(cpu_core_map);
56
57static DECLARE_COMPLETION(cpu_starting);
58static DECLARE_COMPLETION(cpu_running);
59
60/*
61 * A logcal cpu mask containing only one VPE per core to
62 * reduce the number of IPIs on large MT systems.
63 */
64cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly;
65EXPORT_SYMBOL(cpu_foreign_map);
66
67/* representing cpus for which sibling maps can be computed */
68static cpumask_t cpu_sibling_setup_map;
69
70/* representing cpus for which llc shared maps can be computed */
71static cpumask_t cpu_llc_shared_setup_map;
72
73/* representing cpus for which core maps can be computed */
74static cpumask_t cpu_core_setup_map;
75
76struct secondary_data cpuboot_data;
77static DEFINE_PER_CPU(int, cpu_state);
78
79static const char *ipi_types[NR_IPI] __tracepoint_string = {
80 [IPI_RESCHEDULE] = "Rescheduling interrupts",
81 [IPI_CALL_FUNCTION] = "Function call interrupts",
82 [IPI_IRQ_WORK] = "IRQ work interrupts",
83 [IPI_CLEAR_VECTOR] = "Clear vector interrupts",
84};
85
86void show_ipi_list(struct seq_file *p, int prec)
87{
88 unsigned int cpu, i;
89
90 for (i = 0; i < NR_IPI; i++) {
91 seq_printf(m: p, fmt: "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : "");
92 for_each_online_cpu(cpu)
93 seq_put_decimal_ull_width(m: p, delimiter: " ", per_cpu(irq_stat, cpu).ipi_irqs[i], width: 10);
94 seq_printf(m: p, fmt: " LoongArch %d %s\n", i + 1, ipi_types[i]);
95 }
96}
97
98static inline void set_cpu_core_map(int cpu)
99{
100 int i;
101
102 cpumask_set_cpu(cpu, dstp: &cpu_core_setup_map);
103
104 for_each_cpu(i, &cpu_core_setup_map) {
105 if (cpu_data[cpu].package == cpu_data[i].package) {
106 cpumask_set_cpu(cpu: i, dstp: &cpu_core_map[cpu]);
107 cpumask_set_cpu(cpu, dstp: &cpu_core_map[i]);
108 }
109 }
110}
111
112static inline void set_cpu_llc_shared_map(int cpu)
113{
114 int i;
115
116 cpumask_set_cpu(cpu, dstp: &cpu_llc_shared_setup_map);
117
118 for_each_cpu(i, &cpu_llc_shared_setup_map) {
119 if (cpu_to_node(cpu) == cpu_to_node(cpu: i)) {
120 cpumask_set_cpu(cpu: i, dstp: &cpu_llc_shared_map[cpu]);
121 cpumask_set_cpu(cpu, dstp: &cpu_llc_shared_map[i]);
122 }
123 }
124}
125
126static inline void clear_cpu_llc_shared_map(int cpu)
127{
128 int i;
129
130 for_each_cpu(i, &cpu_llc_shared_setup_map) {
131 if (cpu_to_node(cpu) == cpu_to_node(cpu: i)) {
132 cpumask_clear_cpu(cpu: i, dstp: &cpu_llc_shared_map[cpu]);
133 cpumask_clear_cpu(cpu, dstp: &cpu_llc_shared_map[i]);
134 }
135 }
136
137 cpumask_clear_cpu(cpu, dstp: &cpu_llc_shared_setup_map);
138}
139
140static inline void set_cpu_sibling_map(int cpu)
141{
142 int i;
143
144 cpumask_set_cpu(cpu, dstp: &cpu_sibling_setup_map);
145
146 for_each_cpu(i, &cpu_sibling_setup_map) {
147 if (cpus_are_siblings(cpu, i)) {
148 cpumask_set_cpu(cpu: i, dstp: &cpu_sibling_map[cpu]);
149 cpumask_set_cpu(cpu, dstp: &cpu_sibling_map[i]);
150 }
151 }
152}
153
154static inline void clear_cpu_sibling_map(int cpu)
155{
156 int i;
157
158 for_each_cpu(i, &cpu_sibling_setup_map) {
159 if (cpus_are_siblings(cpu, i)) {
160 cpumask_clear_cpu(cpu: i, dstp: &cpu_sibling_map[cpu]);
161 cpumask_clear_cpu(cpu, dstp: &cpu_sibling_map[i]);
162 }
163 }
164
165 cpumask_clear_cpu(cpu, dstp: &cpu_sibling_setup_map);
166}
167
168/*
169 * Calculate a new cpu_foreign_map mask whenever a
170 * new cpu appears or disappears.
171 */
172void calculate_cpu_foreign_map(void)
173{
174 int i, k, core_present;
175 cpumask_t temp_foreign_map;
176
177 /* Re-calculate the mask */
178 cpumask_clear(dstp: &temp_foreign_map);
179 for_each_online_cpu(i) {
180 core_present = 0;
181 for_each_cpu(k, &temp_foreign_map)
182 if (cpus_are_siblings(i, k))
183 core_present = 1;
184 if (!core_present)
185 cpumask_set_cpu(cpu: i, dstp: &temp_foreign_map);
186 }
187
188 for_each_online_cpu(i)
189 cpumask_andnot(dstp: &cpu_foreign_map[i],
190 src1p: &temp_foreign_map, src2p: &cpu_sibling_map[i]);
191}
192
193/* Send mailbox buffer via Mail_Send */
194static void csr_mail_send(uint64_t data, int cpu, int mailbox)
195{
196 uint64_t val;
197
198 /* Send high 32 bits */
199 val = IOCSR_MBUF_SEND_BLOCKING;
200 val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT);
201 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT);
202 val |= (data & IOCSR_MBUF_SEND_H32_MASK);
203 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND);
204
205 /* Send low 32 bits */
206 val = IOCSR_MBUF_SEND_BLOCKING;
207 val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT);
208 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT);
209 val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT);
210 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND);
211};
212
213static u32 ipi_read_clear(int cpu)
214{
215 u32 action;
216
217 /* Load the ipi register to figure out what we're supposed to do */
218 action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS);
219 /* Clear the ipi register to clear the interrupt */
220 iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR);
221 wbflush();
222
223 return action;
224}
225
226static void ipi_write_action(int cpu, u32 action)
227{
228 uint32_t val;
229
230 val = IOCSR_IPI_SEND_BLOCKING | action;
231 val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT);
232 iocsr_write32(val, LOONGARCH_IOCSR_IPI_SEND);
233}
234
235static void loongson_send_ipi_single(int cpu, unsigned int action)
236{
237 ipi_write_action(cpu: cpu_logical_map(cpu), action: (u32)action);
238}
239
240static void loongson_send_ipi_mask(const struct cpumask *mask, unsigned int action)
241{
242 unsigned int i;
243
244 for_each_cpu(i, mask)
245 ipi_write_action(cpu: cpu_logical_map(i), action: (u32)action);
246}
247
248/*
249 * This function sends a 'reschedule' IPI to another CPU.
250 * it goes straight through and wastes no time serializing
251 * anything. Worst case is that we lose a reschedule ...
252 */
253void arch_smp_send_reschedule(int cpu)
254{
255 mp_ops.send_ipi_single(cpu, ACTION_RESCHEDULE);
256}
257EXPORT_SYMBOL_GPL(arch_smp_send_reschedule);
258
259#ifdef CONFIG_IRQ_WORK
260void arch_irq_work_raise(void)
261{
262 mp_ops.send_ipi_single(smp_processor_id(), ACTION_IRQ_WORK);
263}
264#endif
265
266static irqreturn_t loongson_ipi_interrupt(int irq, void *dev)
267{
268 unsigned int action;
269 unsigned int cpu = smp_processor_id();
270
271 action = ipi_read_clear(cpu: cpu_logical_map(cpu));
272
273 if (action & SMP_RESCHEDULE) {
274 scheduler_ipi();
275 per_cpu(irq_stat, cpu).ipi_irqs[IPI_RESCHEDULE]++;
276 }
277
278 if (action & SMP_CALL_FUNCTION) {
279 generic_smp_call_function_interrupt();
280 per_cpu(irq_stat, cpu).ipi_irqs[IPI_CALL_FUNCTION]++;
281 }
282
283 if (action & SMP_IRQ_WORK) {
284 irq_work_run();
285 per_cpu(irq_stat, cpu).ipi_irqs[IPI_IRQ_WORK]++;
286 }
287
288 if (action & SMP_CLEAR_VECTOR) {
289 complete_irq_moving();
290 per_cpu(irq_stat, cpu).ipi_irqs[IPI_CLEAR_VECTOR]++;
291 }
292
293 return IRQ_HANDLED;
294}
295
296static void loongson_init_ipi(void)
297{
298 int r, ipi_irq;
299
300 ipi_irq = get_percpu_irq(INT_IPI);
301 if (ipi_irq < 0)
302 panic(fmt: "IPI IRQ mapping failed\n");
303
304 irq_set_percpu_devid(ipi_irq);
305 r = request_percpu_irq(irq: ipi_irq, handler: loongson_ipi_interrupt, devname: "IPI", percpu_dev_id: &irq_stat);
306 if (r < 0)
307 panic(fmt: "IPI IRQ request failed\n");
308}
309
310struct smp_ops mp_ops = {
311 .init_ipi = loongson_init_ipi,
312 .send_ipi_single = loongson_send_ipi_single,
313 .send_ipi_mask = loongson_send_ipi_mask,
314};
315
316static void __init fdt_smp_setup(void)
317{
318#ifdef CONFIG_OF
319 unsigned int cpu, cpuid;
320 struct device_node *node = NULL;
321
322 for_each_of_cpu_node(node) {
323 if (!of_device_is_available(device: node))
324 continue;
325
326 cpuid = of_get_cpu_hwid(cpun: node, thread: 0);
327 if (cpuid >= nr_cpu_ids)
328 continue;
329
330 if (cpuid == loongson_sysconf.boot_cpu_id)
331 cpu = 0;
332 else
333 cpu = find_first_zero_bit(cpumask_bits(cpu_present_mask), NR_CPUS);
334
335 num_processors++;
336 set_cpu_possible(cpu, possible: true);
337 set_cpu_present(cpu, true);
338 __cpu_number_map[cpuid] = cpu;
339 __cpu_logical_map[cpu] = cpuid;
340
341 early_numa_add_cpu(cpuid, 0);
342 set_cpuid_to_node(cpuid, 0);
343 }
344
345 loongson_sysconf.nr_cpus = num_processors;
346 set_bit(0, loongson_sysconf.cores_io_master);
347#endif
348}
349
350void __init loongson_smp_setup(void)
351{
352 fdt_smp_setup();
353
354 if (loongson_sysconf.cores_per_package == 0)
355 loongson_sysconf.cores_per_package = num_processors;
356
357 cpu_data[0].core = cpu_logical_map(0) % loongson_sysconf.cores_per_package;
358 cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package;
359
360 pv_ipi_init();
361 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
362 pr_info("Detected %i available CPU(s)\n", loongson_sysconf.nr_cpus);
363}
364
365void __init loongson_prepare_cpus(unsigned int max_cpus)
366{
367 int i = 0;
368
369 parse_acpi_topology();
370 cpu_data[0].global_id = cpu_logical_map(0);
371
372 for (i = 0; i < loongson_sysconf.nr_cpus; i++) {
373 set_cpu_present(i, true);
374 csr_mail_send(0, __cpu_logical_map[i], 0);
375 }
376
377 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
378}
379
380/*
381 * Setup the PC, SP, and TP of a secondary processor and start it running!
382 */
383void loongson_boot_secondary(int cpu, struct task_struct *idle)
384{
385 unsigned long entry;
386
387 pr_info("Booting CPU#%d...\n", cpu);
388
389 entry = __pa_symbol((unsigned long)&smpboot_entry);
390 cpuboot_data.stack = (unsigned long)__KSTK_TOS(idle);
391 cpuboot_data.thread_info = (unsigned long)task_thread_info(idle);
392
393 csr_mail_send(data: entry, cpu: cpu_logical_map(cpu), mailbox: 0);
394
395 loongson_send_ipi_single(cpu, ACTION_BOOT_CPU);
396}
397
398/*
399 * SMP init and finish on secondary CPUs
400 */
401void loongson_init_secondary(void)
402{
403 unsigned int cpu = smp_processor_id();
404 unsigned int imask = ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 |
405 ECFGF_IPI | ECFGF_PMC | ECFGF_TIMER | ECFGF_SIP0;
406
407 change_csr_ecfg(ECFG0_IM, imask);
408
409 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
410
411#ifdef CONFIG_NUMA
412 numa_add_cpu(cpu);
413#endif
414 per_cpu(cpu_state, cpu) = CPU_ONLINE;
415 cpu_data[cpu].package =
416 cpu_logical_map(cpu) / loongson_sysconf.cores_per_package;
417 cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core :
418 cpu_logical_map(cpu) % loongson_sysconf.cores_per_package;
419 cpu_data[cpu].global_id = cpu_logical_map(cpu);
420}
421
422void loongson_smp_finish(void)
423{
424 local_irq_enable();
425 iocsr_write64(0, LOONGARCH_IOCSR_MBUF0);
426 pr_info("CPU#%d finished\n", smp_processor_id());
427}
428
429#ifdef CONFIG_HOTPLUG_CPU
430
431int loongson_cpu_disable(void)
432{
433 unsigned long flags;
434 unsigned int cpu = smp_processor_id();
435
436 if (io_master(cpu))
437 return -EBUSY;
438
439#ifdef CONFIG_NUMA
440 numa_remove_cpu(cpu);
441#endif
442 set_cpu_online(cpu, online: false);
443 clear_cpu_sibling_map(cpu);
444 clear_cpu_llc_shared_map(cpu);
445 calculate_cpu_foreign_map();
446 local_irq_save(flags);
447 irq_migrate_all_off_this_cpu();
448 clear_csr_ecfg(ECFG0_IM);
449 local_irq_restore(flags);
450 local_flush_tlb_all();
451
452 return 0;
453}
454
455void loongson_cpu_die(unsigned int cpu)
456{
457 while (per_cpu(cpu_state, cpu) != CPU_DEAD)
458 cpu_relax();
459
460 mb();
461}
462
463static void __noreturn idle_play_dead(void)
464{
465 register uint64_t addr;
466 register void (*init_fn)(void);
467
468 idle_task_exit();
469 local_irq_enable();
470 set_csr_ecfg(ECFGF_IPI);
471 __this_cpu_write(cpu_state, CPU_DEAD);
472
473 __smp_mb();
474 do {
475 __asm__ __volatile__("idle 0\n\t");
476 addr = iocsr_read64(LOONGARCH_IOCSR_MBUF0);
477 } while (addr == 0);
478
479 local_irq_disable();
480 init_fn = (void *)TO_CACHE(addr);
481 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR);
482
483 init_fn();
484 BUG();
485}
486
487#ifdef CONFIG_HIBERNATION
488static void __noreturn poll_play_dead(void)
489{
490 register uint64_t addr;
491 register void (*init_fn)(void);
492
493 idle_task_exit();
494 __this_cpu_write(cpu_state, CPU_DEAD);
495
496 __smp_mb();
497 do {
498 __asm__ __volatile__("nop\n\t");
499 addr = iocsr_read64(LOONGARCH_IOCSR_MBUF0);
500 } while (addr == 0);
501
502 init_fn = (void *)TO_CACHE(addr);
503 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR);
504
505 init_fn();
506 BUG();
507}
508#endif
509
510static void (*play_dead)(void) = idle_play_dead;
511
512void __noreturn arch_cpu_idle_dead(void)
513{
514 play_dead();
515 BUG(); /* play_dead() doesn't return */
516}
517
518#ifdef CONFIG_HIBERNATION
519int hibernate_resume_nonboot_cpu_disable(void)
520{
521 int ret;
522
523 play_dead = poll_play_dead;
524 ret = suspend_disable_secondary_cpus();
525 play_dead = idle_play_dead;
526
527 return ret;
528}
529#endif
530
531#endif
532
533/*
534 * Power management
535 */
536#ifdef CONFIG_PM
537
538static int loongson_ipi_suspend(void *data)
539{
540 return 0;
541}
542
543static void loongson_ipi_resume(void *data)
544{
545 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
546}
547
548static const struct syscore_ops loongson_ipi_syscore_ops = {
549 .resume = loongson_ipi_resume,
550 .suspend = loongson_ipi_suspend,
551};
552
553static struct syscore loongson_ipi_syscore = {
554 .ops = &loongson_ipi_syscore_ops,
555};
556
557/*
558 * Enable boot cpu ipi before enabling nonboot cpus
559 * during syscore_resume.
560 */
561static int __init ipi_pm_init(void)
562{
563 register_syscore(syscore: &loongson_ipi_syscore);
564 return 0;
565}
566
567core_initcall(ipi_pm_init);
568#endif
569
570/* Preload SMP state for boot cpu */
571void __init smp_prepare_boot_cpu(void)
572{
573 unsigned int cpu, node, rr_node;
574
575 set_cpu_possible(cpu: 0, possible: true);
576 set_cpu_online(cpu: 0, online: true);
577 set_my_cpu_offset(per_cpu_offset(0));
578 numa_add_cpu(cpu: 0);
579
580 rr_node = first_node(node_online_map);
581 for_each_possible_cpu(cpu) {
582 node = early_cpu_to_node(cpu);
583
584 /*
585 * The mapping between present cpus and nodes has been
586 * built during MADT and SRAT parsing.
587 *
588 * If possible cpus = present cpus here, early_cpu_to_node
589 * will return valid node.
590 *
591 * If possible cpus > present cpus here (e.g. some possible
592 * cpus will be added by cpu-hotplug later), for possible but
593 * not present cpus, early_cpu_to_node will return NUMA_NO_NODE,
594 * and we just map them to online nodes in round-robin way.
595 * Once hotplugged, new correct mapping will be built for them.
596 */
597 if (node != NUMA_NO_NODE)
598 set_cpu_numa_node(cpu, node);
599 else {
600 set_cpu_numa_node(cpu, node: rr_node);
601 rr_node = next_node_in(rr_node, node_online_map);
602 }
603 }
604
605 pv_spinlock_init();
606}
607
608/* called from main before smp_init() */
609void __init smp_prepare_cpus(unsigned int max_cpus)
610{
611 init_new_context(current, mm: &init_mm);
612 current_thread_info()->cpu = 0;
613 loongson_prepare_cpus(max_cpus);
614 set_cpu_sibling_map(0);
615 set_cpu_llc_shared_map(0);
616 set_cpu_core_map(0);
617 calculate_cpu_foreign_map();
618#ifndef CONFIG_HOTPLUG_CPU
619 init_cpu_present(cpu_possible_mask);
620#endif
621}
622
623int __cpu_up(unsigned int cpu, struct task_struct *tidle)
624{
625 loongson_boot_secondary(cpu, idle: tidle);
626
627 /* Wait for CPU to start and be ready to sync counters */
628 if (!wait_for_completion_timeout(x: &cpu_starting,
629 timeout: msecs_to_jiffies(m: 5000))) {
630 pr_crit("CPU%u: failed to start\n", cpu);
631 return -EIO;
632 }
633
634 /* Wait for CPU to finish startup & mark itself online before return */
635 wait_for_completion(&cpu_running);
636
637 return 0;
638}
639
640/*
641 * First C code run on the secondary CPUs after being started up by
642 * the master.
643 */
644asmlinkage void start_secondary(void)
645{
646 unsigned int cpu;
647
648 sync_counter();
649 cpu = raw_smp_processor_id();
650 set_my_cpu_offset(per_cpu_offset(cpu));
651
652 cpu_probe();
653 constant_clockevent_init();
654 loongson_init_secondary();
655
656 set_cpu_sibling_map(cpu);
657 set_cpu_llc_shared_map(cpu);
658 set_cpu_core_map(cpu);
659
660 notify_cpu_starting(cpu);
661
662 /* Notify boot CPU that we're starting */
663 complete(&cpu_starting);
664
665 /* The CPU is running, now mark it online */
666 set_cpu_online(cpu, online: true);
667
668 calculate_cpu_foreign_map();
669
670 /*
671 * Notify boot CPU that we're up & online and it can safely return
672 * from __cpu_up()
673 */
674 complete(&cpu_running);
675
676 /*
677 * irq will be enabled in loongson_smp_finish(), enabling it too
678 * early is dangerous.
679 */
680 WARN_ON_ONCE(!irqs_disabled());
681 loongson_smp_finish();
682
683 cpu_startup_entry(state: CPUHP_AP_ONLINE_IDLE);
684}
685
686void __init smp_cpus_done(unsigned int max_cpus)
687{
688}
689
690static void stop_this_cpu(void *dummy)
691{
692 set_cpu_online(smp_processor_id(), online: false);
693 calculate_cpu_foreign_map();
694 local_irq_disable();
695 while (true);
696}
697
698void smp_send_stop(void)
699{
700 smp_call_function(func: stop_this_cpu, NULL, wait: 0);
701}
702
703#ifdef CONFIG_PROFILING
704int setup_profiling_timer(unsigned int multiplier)
705{
706 return 0;
707}
708#endif
709
710static void flush_tlb_all_ipi(void *info)
711{
712 local_flush_tlb_all();
713}
714
715void flush_tlb_all(void)
716{
717 on_each_cpu(func: flush_tlb_all_ipi, NULL, wait: 1);
718}
719
720static void flush_tlb_mm_ipi(void *mm)
721{
722 local_flush_tlb_mm((struct mm_struct *)mm);
723}
724
725void flush_tlb_mm(struct mm_struct *mm)
726{
727 if (atomic_read(v: &mm->mm_users) == 0)
728 return; /* happens as a result of exit_mmap() */
729
730 preempt_disable();
731
732 if ((atomic_read(v: &mm->mm_users) != 1) || (current->mm != mm)) {
733 on_each_cpu_mask(mask: mm_cpumask(mm), func: flush_tlb_mm_ipi, info: mm, wait: 1);
734 } else {
735 unsigned int cpu;
736
737 for_each_online_cpu(cpu) {
738 if (cpu != smp_processor_id() && cpu_context(cpu, mm))
739 cpu_context(cpu, mm) = 0;
740 }
741 local_flush_tlb_mm(mm);
742 }
743
744 preempt_enable();
745}
746
747struct flush_tlb_data {
748 struct vm_area_struct *vma;
749 unsigned long addr1;
750 unsigned long addr2;
751};
752
753static void flush_tlb_range_ipi(void *info)
754{
755 struct flush_tlb_data *fd = info;
756
757 local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
758}
759
760void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
761{
762 struct mm_struct *mm = vma->vm_mm;
763
764 preempt_disable();
765 if ((atomic_read(v: &mm->mm_users) != 1) || (current->mm != mm)) {
766 struct flush_tlb_data fd = {
767 .vma = vma,
768 .addr1 = start,
769 .addr2 = end,
770 };
771
772 on_each_cpu_mask(mask: mm_cpumask(mm), func: flush_tlb_range_ipi, info: &fd, wait: 1);
773 } else {
774 unsigned int cpu;
775
776 for_each_online_cpu(cpu) {
777 if (cpu != smp_processor_id() && cpu_context(cpu, mm))
778 cpu_context(cpu, mm) = 0;
779 }
780 local_flush_tlb_range(vma, start, end);
781 }
782 preempt_enable();
783}
784
785static void flush_tlb_kernel_range_ipi(void *info)
786{
787 struct flush_tlb_data *fd = info;
788
789 local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
790}
791
792void flush_tlb_kernel_range(unsigned long start, unsigned long end)
793{
794 struct flush_tlb_data fd = {
795 .addr1 = start,
796 .addr2 = end,
797 };
798
799 on_each_cpu(func: flush_tlb_kernel_range_ipi, info: &fd, wait: 1);
800}
801
802static void flush_tlb_page_ipi(void *info)
803{
804 struct flush_tlb_data *fd = info;
805
806 local_flush_tlb_page(fd->vma, fd->addr1);
807}
808
809void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
810{
811 preempt_disable();
812 if ((atomic_read(v: &vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
813 struct flush_tlb_data fd = {
814 .vma = vma,
815 .addr1 = page,
816 };
817
818 on_each_cpu_mask(mask: mm_cpumask(mm: vma->vm_mm), func: flush_tlb_page_ipi, info: &fd, wait: 1);
819 } else {
820 unsigned int cpu;
821
822 for_each_online_cpu(cpu) {
823 if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm))
824 cpu_context(cpu, vma->vm_mm) = 0;
825 }
826 local_flush_tlb_page(vma, page);
827 }
828 preempt_enable();
829}
830EXPORT_SYMBOL(flush_tlb_page);
831
832static void flush_tlb_one_ipi(void *info)
833{
834 unsigned long vaddr = (unsigned long) info;
835
836 local_flush_tlb_one(vaddr);
837}
838
839void flush_tlb_one(unsigned long vaddr)
840{
841 on_each_cpu(func: flush_tlb_one_ipi, info: (void *)vaddr, wait: 1);
842}
843EXPORT_SYMBOL(flush_tlb_one);
844

source code of linux/arch/loongarch/kernel/smp.c