1// SPDX-License-Identifier: GPL-2.0+
2//
3// Scalability test comparing RCU vs other mechanisms
4// for acquiring references on objects.
5//
6// Copyright (C) Google, 2020.
7//
8// Author: Joel Fernandes <joel@joelfernandes.org>
9
10#define pr_fmt(fmt) fmt
11
12#include <linux/atomic.h>
13#include <linux/bitops.h>
14#include <linux/completion.h>
15#include <linux/cpu.h>
16#include <linux/delay.h>
17#include <linux/err.h>
18#include <linux/init.h>
19#include <linux/interrupt.h>
20#include <linux/kthread.h>
21#include <linux/kernel.h>
22#include <linux/mm.h>
23#include <linux/module.h>
24#include <linux/moduleparam.h>
25#include <linux/notifier.h>
26#include <linux/percpu.h>
27#include <linux/rcupdate.h>
28#include <linux/rcupdate_trace.h>
29#include <linux/reboot.h>
30#include <linux/sched.h>
31#include <linux/seq_buf.h>
32#include <linux/spinlock.h>
33#include <linux/smp.h>
34#include <linux/stat.h>
35#include <linux/srcu.h>
36#include <linux/slab.h>
37#include <linux/torture.h>
38#include <linux/types.h>
39#include <linux/sched/clock.h>
40
41#include "rcu.h"
42
43#define SCALE_FLAG "-ref-scale: "
44
45#define SCALEOUT(s, x...) \
46 pr_alert("%s" SCALE_FLAG s, scale_type, ## x)
47
48#define VERBOSE_SCALEOUT(s, x...) \
49 do { \
50 if (verbose) \
51 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
52 } while (0)
53
54static atomic_t verbose_batch_ctr;
55
56#define VERBOSE_SCALEOUT_BATCH(s, x...) \
57do { \
58 if (verbose && \
59 (verbose_batched <= 0 || \
60 !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \
61 schedule_timeout_uninterruptible(1); \
62 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
63 } \
64} while (0)
65
66#define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x)
67
68MODULE_DESCRIPTION("Scalability test for object reference mechanisms");
69MODULE_LICENSE("GPL");
70MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");
71
72static char *scale_type = "rcu";
73module_param(scale_type, charp, 0444);
74MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock.");
75
76torture_param(int, verbose, 0, "Enable verbose debugging printk()s");
77torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s");
78
79// Number of seconds to extend warm-up and cool-down for multiple guest OSes
80torture_param(long, guest_os_delay, 0,
81 "Number of seconds to extend warm-up/cool-down for multiple guest OSes.");
82// Wait until there are multiple CPUs before starting test.
83torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
84 "Holdoff time before test start (s)");
85// Number of typesafe_lookup structures, that is, the degree of concurrency.
86torture_param(long, lookup_instances, 0, "Number of typesafe_lookup structures.");
87// Number of loops per experiment, all readers execute operations concurrently.
88torture_param(int, loops, 10000, "Number of loops per experiment.");
89// Number of readers, with -1 defaulting to about 75% of the CPUs.
90torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
91// Number of runs.
92torture_param(int, nruns, 30, "Number of experiments to run.");
93// Reader delay in nanoseconds, 0 for no delay.
94torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
95
96#ifdef MODULE
97# define REFSCALE_SHUTDOWN 0
98#else
99# define REFSCALE_SHUTDOWN 1
100#endif
101
102torture_param(bool, shutdown, REFSCALE_SHUTDOWN,
103 "Shutdown at end of scalability tests.");
104
105struct reader_task {
106 struct task_struct *task;
107 int start_reader;
108 wait_queue_head_t wq;
109 u64 last_duration_ns;
110};
111
112static struct task_struct *shutdown_task;
113static wait_queue_head_t shutdown_wq;
114
115static struct task_struct *main_task;
116static wait_queue_head_t main_wq;
117static int shutdown_start;
118
119static struct reader_task *reader_tasks;
120
121// Number of readers that are part of the current experiment.
122static atomic_t nreaders_exp;
123
124// Use to wait for all threads to start.
125static atomic_t n_init;
126static atomic_t n_started;
127static atomic_t n_warmedup;
128static atomic_t n_cooleddown;
129
130// Track which experiment is currently running.
131static int exp_idx;
132
133// Operations vector for selecting different types of tests.
134struct ref_scale_ops {
135 bool (*init)(void);
136 void (*cleanup)(void);
137 void (*readsection)(const int nloops);
138 void (*delaysection)(const int nloops, const int udl, const int ndl);
139 bool enable_irqs;
140 const char *name;
141};
142
143static const struct ref_scale_ops *cur_ops;
144
145static void un_delay(const int udl, const int ndl)
146{
147 if (udl)
148 udelay(usec: udl);
149 if (ndl)
150 ndelay(ndl);
151}
152
153static void ref_rcu_read_section(const int nloops)
154{
155 int i;
156
157 for (i = nloops; i >= 0; i--) {
158 rcu_read_lock();
159 rcu_read_unlock();
160 }
161}
162
163static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl)
164{
165 int i;
166
167 for (i = nloops; i >= 0; i--) {
168 rcu_read_lock();
169 un_delay(udl, ndl);
170 rcu_read_unlock();
171 }
172}
173
174static bool rcu_sync_scale_init(void)
175{
176 return true;
177}
178
179static const struct ref_scale_ops rcu_ops = {
180 .init = rcu_sync_scale_init,
181 .readsection = ref_rcu_read_section,
182 .delaysection = ref_rcu_delay_section,
183 .name = "rcu"
184};
185
186// Definitions for SRCU ref scale testing.
187DEFINE_STATIC_SRCU(srcu_refctl_scale);
188DEFINE_STATIC_SRCU_FAST(srcu_fast_refctl_scale);
189DEFINE_STATIC_SRCU_FAST_UPDOWN(srcu_fast_updown_refctl_scale);
190static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale;
191
192static void srcu_ref_scale_read_section(const int nloops)
193{
194 int i;
195 int idx;
196
197 for (i = nloops; i >= 0; i--) {
198 idx = srcu_read_lock(ssp: srcu_ctlp);
199 srcu_read_unlock(ssp: srcu_ctlp, idx);
200 }
201}
202
203static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
204{
205 int i;
206 int idx;
207
208 for (i = nloops; i >= 0; i--) {
209 idx = srcu_read_lock(ssp: srcu_ctlp);
210 un_delay(udl, ndl);
211 srcu_read_unlock(ssp: srcu_ctlp, idx);
212 }
213}
214
215static const struct ref_scale_ops srcu_ops = {
216 .init = rcu_sync_scale_init,
217 .readsection = srcu_ref_scale_read_section,
218 .delaysection = srcu_ref_scale_delay_section,
219 .name = "srcu"
220};
221
222static bool srcu_fast_sync_scale_init(void)
223{
224 srcu_ctlp = &srcu_fast_refctl_scale;
225 return true;
226}
227
228static void srcu_fast_ref_scale_read_section(const int nloops)
229{
230 int i;
231 struct srcu_ctr __percpu *scp;
232
233 for (i = nloops; i >= 0; i--) {
234 scp = srcu_read_lock_fast(ssp: srcu_ctlp);
235 srcu_read_unlock_fast(ssp: srcu_ctlp, scp);
236 }
237}
238
239static void srcu_fast_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
240{
241 int i;
242 struct srcu_ctr __percpu *scp;
243
244 for (i = nloops; i >= 0; i--) {
245 scp = srcu_read_lock_fast(ssp: srcu_ctlp);
246 un_delay(udl, ndl);
247 srcu_read_unlock_fast(ssp: srcu_ctlp, scp);
248 }
249}
250
251static const struct ref_scale_ops srcu_fast_ops = {
252 .init = srcu_fast_sync_scale_init,
253 .readsection = srcu_fast_ref_scale_read_section,
254 .delaysection = srcu_fast_ref_scale_delay_section,
255 .name = "srcu-fast"
256};
257
258static bool srcu_fast_updown_sync_scale_init(void)
259{
260 srcu_ctlp = &srcu_fast_updown_refctl_scale;
261 return true;
262}
263
264static void srcu_fast_updown_ref_scale_read_section(const int nloops)
265{
266 int i;
267 struct srcu_ctr __percpu *scp;
268
269 for (i = nloops; i >= 0; i--) {
270 scp = srcu_read_lock_fast_updown(ssp: srcu_ctlp);
271 srcu_read_unlock_fast_updown(ssp: srcu_ctlp, scp);
272 }
273}
274
275static void srcu_fast_updown_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
276{
277 int i;
278 struct srcu_ctr __percpu *scp;
279
280 for (i = nloops; i >= 0; i--) {
281 scp = srcu_read_lock_fast_updown(ssp: srcu_ctlp);
282 un_delay(udl, ndl);
283 srcu_read_unlock_fast_updown(ssp: srcu_ctlp, scp);
284 }
285}
286
287static const struct ref_scale_ops srcu_fast_updown_ops = {
288 .init = srcu_fast_updown_sync_scale_init,
289 .readsection = srcu_fast_updown_ref_scale_read_section,
290 .delaysection = srcu_fast_updown_ref_scale_delay_section,
291 .name = "srcu-fast-updown"
292};
293
294#ifdef CONFIG_TASKS_RCU
295
296// Definitions for RCU Tasks ref scale testing: Empty read markers.
297// These definitions also work for RCU Rude readers.
298static void rcu_tasks_ref_scale_read_section(const int nloops)
299{
300 int i;
301
302 for (i = nloops; i >= 0; i--)
303 continue;
304}
305
306static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
307{
308 int i;
309
310 for (i = nloops; i >= 0; i--)
311 un_delay(udl, ndl);
312}
313
314static const struct ref_scale_ops rcu_tasks_ops = {
315 .init = rcu_sync_scale_init,
316 .readsection = rcu_tasks_ref_scale_read_section,
317 .delaysection = rcu_tasks_ref_scale_delay_section,
318 .name = "rcu-tasks"
319};
320
321#define RCU_TASKS_OPS &rcu_tasks_ops,
322
323#else // #ifdef CONFIG_TASKS_RCU
324
325#define RCU_TASKS_OPS
326
327#endif // #else // #ifdef CONFIG_TASKS_RCU
328
329#ifdef CONFIG_TASKS_TRACE_RCU
330
331// Definitions for RCU Tasks Trace ref scale testing.
332static void rcu_trace_ref_scale_read_section(const int nloops)
333{
334 int i;
335
336 for (i = nloops; i >= 0; i--) {
337 rcu_read_lock_trace();
338 rcu_read_unlock_trace();
339 }
340}
341
342static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
343{
344 int i;
345
346 for (i = nloops; i >= 0; i--) {
347 rcu_read_lock_trace();
348 un_delay(udl, ndl);
349 rcu_read_unlock_trace();
350 }
351}
352
353static const struct ref_scale_ops rcu_trace_ops = {
354 .init = rcu_sync_scale_init,
355 .readsection = rcu_trace_ref_scale_read_section,
356 .delaysection = rcu_trace_ref_scale_delay_section,
357 .name = "rcu-trace"
358};
359
360#define RCU_TRACE_OPS &rcu_trace_ops,
361
362#else // #ifdef CONFIG_TASKS_TRACE_RCU
363
364#define RCU_TRACE_OPS
365
366#endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU
367
368// Definitions for reference count
369static atomic_t refcnt;
370
371// Definitions acquire-release.
372static DEFINE_PER_CPU(unsigned long, test_acqrel);
373
374static void ref_refcnt_section(const int nloops)
375{
376 int i;
377
378 for (i = nloops; i >= 0; i--) {
379 atomic_inc(v: &refcnt);
380 atomic_dec(v: &refcnt);
381 }
382}
383
384static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl)
385{
386 int i;
387
388 for (i = nloops; i >= 0; i--) {
389 atomic_inc(v: &refcnt);
390 un_delay(udl, ndl);
391 atomic_dec(v: &refcnt);
392 }
393}
394
395static const struct ref_scale_ops refcnt_ops = {
396 .init = rcu_sync_scale_init,
397 .readsection = ref_refcnt_section,
398 .delaysection = ref_refcnt_delay_section,
399 .name = "refcnt"
400};
401
402static void ref_percpuinc_section(const int nloops)
403{
404 int i;
405
406 for (i = nloops; i >= 0; i--) {
407 this_cpu_inc(test_acqrel);
408 this_cpu_dec(test_acqrel);
409 }
410}
411
412static void ref_percpuinc_delay_section(const int nloops, const int udl, const int ndl)
413{
414 int i;
415
416 for (i = nloops; i >= 0; i--) {
417 this_cpu_inc(test_acqrel);
418 un_delay(udl, ndl);
419 this_cpu_dec(test_acqrel);
420 }
421}
422
423static const struct ref_scale_ops percpuinc_ops = {
424 .init = rcu_sync_scale_init,
425 .readsection = ref_percpuinc_section,
426 .delaysection = ref_percpuinc_delay_section,
427 .name = "percpuinc"
428};
429
430// Note that this can lose counts in preemptible kernels.
431static void ref_incpercpu_section(const int nloops)
432{
433 int i;
434
435 for (i = nloops; i >= 0; i--) {
436 unsigned long *tap = this_cpu_ptr(&test_acqrel);
437
438 WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
439 WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
440 }
441}
442
443static void ref_incpercpu_delay_section(const int nloops, const int udl, const int ndl)
444{
445 int i;
446
447 for (i = nloops; i >= 0; i--) {
448 unsigned long *tap = this_cpu_ptr(&test_acqrel);
449
450 WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
451 un_delay(udl, ndl);
452 WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
453 }
454}
455
456static const struct ref_scale_ops incpercpu_ops = {
457 .init = rcu_sync_scale_init,
458 .readsection = ref_incpercpu_section,
459 .delaysection = ref_incpercpu_delay_section,
460 .name = "incpercpu"
461};
462
463static void ref_incpercpupreempt_section(const int nloops)
464{
465 int i;
466
467 for (i = nloops; i >= 0; i--) {
468 unsigned long *tap;
469
470 preempt_disable();
471 tap = this_cpu_ptr(&test_acqrel);
472 WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
473 WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
474 preempt_enable();
475 }
476}
477
478static void ref_incpercpupreempt_delay_section(const int nloops, const int udl, const int ndl)
479{
480 int i;
481
482 for (i = nloops; i >= 0; i--) {
483 unsigned long *tap;
484
485 preempt_disable();
486 tap = this_cpu_ptr(&test_acqrel);
487 WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
488 un_delay(udl, ndl);
489 WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
490 preempt_enable();
491 }
492}
493
494static const struct ref_scale_ops incpercpupreempt_ops = {
495 .init = rcu_sync_scale_init,
496 .readsection = ref_incpercpupreempt_section,
497 .delaysection = ref_incpercpupreempt_delay_section,
498 .name = "incpercpupreempt"
499};
500
501static void ref_incpercpubh_section(const int nloops)
502{
503 int i;
504
505 for (i = nloops; i >= 0; i--) {
506 unsigned long *tap;
507
508 local_bh_disable();
509 tap = this_cpu_ptr(&test_acqrel);
510 WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
511 WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
512 local_bh_enable();
513 }
514}
515
516static void ref_incpercpubh_delay_section(const int nloops, const int udl, const int ndl)
517{
518 int i;
519
520 for (i = nloops; i >= 0; i--) {
521 unsigned long *tap;
522
523 local_bh_disable();
524 tap = this_cpu_ptr(&test_acqrel);
525 WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
526 un_delay(udl, ndl);
527 WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
528 local_bh_enable();
529 }
530}
531
532static const struct ref_scale_ops incpercpubh_ops = {
533 .init = rcu_sync_scale_init,
534 .readsection = ref_incpercpubh_section,
535 .delaysection = ref_incpercpubh_delay_section,
536 .enable_irqs = true,
537 .name = "incpercpubh"
538};
539
540static void ref_incpercpuirqsave_section(const int nloops)
541{
542 int i;
543 unsigned long flags;
544
545 for (i = nloops; i >= 0; i--) {
546 unsigned long *tap;
547
548 local_irq_save(flags);
549 tap = this_cpu_ptr(&test_acqrel);
550 WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
551 WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
552 local_irq_restore(flags);
553 }
554}
555
556static void ref_incpercpuirqsave_delay_section(const int nloops, const int udl, const int ndl)
557{
558 int i;
559 unsigned long flags;
560
561 for (i = nloops; i >= 0; i--) {
562 unsigned long *tap;
563
564 local_irq_save(flags);
565 tap = this_cpu_ptr(&test_acqrel);
566 WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);
567 un_delay(udl, ndl);
568 WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);
569 local_irq_restore(flags);
570 }
571}
572
573static const struct ref_scale_ops incpercpuirqsave_ops = {
574 .init = rcu_sync_scale_init,
575 .readsection = ref_incpercpuirqsave_section,
576 .delaysection = ref_incpercpuirqsave_delay_section,
577 .name = "incpercpuirqsave"
578};
579
580// Definitions for rwlock
581static rwlock_t test_rwlock;
582
583static bool ref_rwlock_init(void)
584{
585 rwlock_init(&test_rwlock);
586 return true;
587}
588
589static void ref_rwlock_section(const int nloops)
590{
591 int i;
592
593 for (i = nloops; i >= 0; i--) {
594 read_lock(&test_rwlock);
595 read_unlock(&test_rwlock);
596 }
597}
598
599static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl)
600{
601 int i;
602
603 for (i = nloops; i >= 0; i--) {
604 read_lock(&test_rwlock);
605 un_delay(udl, ndl);
606 read_unlock(&test_rwlock);
607 }
608}
609
610static const struct ref_scale_ops rwlock_ops = {
611 .init = ref_rwlock_init,
612 .readsection = ref_rwlock_section,
613 .delaysection = ref_rwlock_delay_section,
614 .name = "rwlock"
615};
616
617// Definitions for rwsem
618static struct rw_semaphore test_rwsem;
619
620static bool ref_rwsem_init(void)
621{
622 init_rwsem(&test_rwsem);
623 return true;
624}
625
626static void ref_rwsem_section(const int nloops)
627{
628 int i;
629
630 for (i = nloops; i >= 0; i--) {
631 down_read(sem: &test_rwsem);
632 up_read(sem: &test_rwsem);
633 }
634}
635
636static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl)
637{
638 int i;
639
640 for (i = nloops; i >= 0; i--) {
641 down_read(sem: &test_rwsem);
642 un_delay(udl, ndl);
643 up_read(sem: &test_rwsem);
644 }
645}
646
647static const struct ref_scale_ops rwsem_ops = {
648 .init = ref_rwsem_init,
649 .readsection = ref_rwsem_section,
650 .delaysection = ref_rwsem_delay_section,
651 .name = "rwsem"
652};
653
654// Definitions for global spinlock
655static DEFINE_RAW_SPINLOCK(test_lock);
656
657static void ref_lock_section(const int nloops)
658{
659 int i;
660
661 preempt_disable();
662 for (i = nloops; i >= 0; i--) {
663 raw_spin_lock(&test_lock);
664 raw_spin_unlock(&test_lock);
665 }
666 preempt_enable();
667}
668
669static void ref_lock_delay_section(const int nloops, const int udl, const int ndl)
670{
671 int i;
672
673 preempt_disable();
674 for (i = nloops; i >= 0; i--) {
675 raw_spin_lock(&test_lock);
676 un_delay(udl, ndl);
677 raw_spin_unlock(&test_lock);
678 }
679 preempt_enable();
680}
681
682static const struct ref_scale_ops lock_ops = {
683 .readsection = ref_lock_section,
684 .delaysection = ref_lock_delay_section,
685 .name = "lock"
686};
687
688// Definitions for global irq-save spinlock
689
690static void ref_lock_irq_section(const int nloops)
691{
692 unsigned long flags;
693 int i;
694
695 preempt_disable();
696 for (i = nloops; i >= 0; i--) {
697 raw_spin_lock_irqsave(&test_lock, flags);
698 raw_spin_unlock_irqrestore(&test_lock, flags);
699 }
700 preempt_enable();
701}
702
703static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl)
704{
705 unsigned long flags;
706 int i;
707
708 preempt_disable();
709 for (i = nloops; i >= 0; i--) {
710 raw_spin_lock_irqsave(&test_lock, flags);
711 un_delay(udl, ndl);
712 raw_spin_unlock_irqrestore(&test_lock, flags);
713 }
714 preempt_enable();
715}
716
717static const struct ref_scale_ops lock_irq_ops = {
718 .readsection = ref_lock_irq_section,
719 .delaysection = ref_lock_irq_delay_section,
720 .name = "lock-irq"
721};
722
723static void ref_acqrel_section(const int nloops)
724{
725 unsigned long x;
726 int i;
727
728 preempt_disable();
729 for (i = nloops; i >= 0; i--) {
730 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
731 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
732 }
733 preempt_enable();
734}
735
736static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl)
737{
738 unsigned long x;
739 int i;
740
741 preempt_disable();
742 for (i = nloops; i >= 0; i--) {
743 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
744 un_delay(udl, ndl);
745 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
746 }
747 preempt_enable();
748}
749
750static const struct ref_scale_ops acqrel_ops = {
751 .readsection = ref_acqrel_section,
752 .delaysection = ref_acqrel_delay_section,
753 .name = "acqrel"
754};
755
756static volatile u64 stopopts;
757
758static void ref_sched_clock_section(const int nloops)
759{
760 u64 x = 0;
761 int i;
762
763 preempt_disable();
764 for (i = nloops; i >= 0; i--)
765 x += sched_clock();
766 preempt_enable();
767 stopopts = x;
768}
769
770static void ref_sched_clock_delay_section(const int nloops, const int udl, const int ndl)
771{
772 u64 x = 0;
773 int i;
774
775 preempt_disable();
776 for (i = nloops; i >= 0; i--) {
777 x += sched_clock();
778 un_delay(udl, ndl);
779 }
780 preempt_enable();
781 stopopts = x;
782}
783
784static const struct ref_scale_ops sched_clock_ops = {
785 .readsection = ref_sched_clock_section,
786 .delaysection = ref_sched_clock_delay_section,
787 .name = "sched-clock"
788};
789
790
791static void ref_clock_section(const int nloops)
792{
793 u64 x = 0;
794 int i;
795
796 preempt_disable();
797 for (i = nloops; i >= 0; i--)
798 x += ktime_get_real_fast_ns();
799 preempt_enable();
800 stopopts = x;
801}
802
803static void ref_clock_delay_section(const int nloops, const int udl, const int ndl)
804{
805 u64 x = 0;
806 int i;
807
808 preempt_disable();
809 for (i = nloops; i >= 0; i--) {
810 x += ktime_get_real_fast_ns();
811 un_delay(udl, ndl);
812 }
813 preempt_enable();
814 stopopts = x;
815}
816
817static const struct ref_scale_ops clock_ops = {
818 .readsection = ref_clock_section,
819 .delaysection = ref_clock_delay_section,
820 .name = "clock"
821};
822
823static void ref_jiffies_section(const int nloops)
824{
825 u64 x = 0;
826 int i;
827
828 preempt_disable();
829 for (i = nloops; i >= 0; i--)
830 x += jiffies;
831 preempt_enable();
832 stopopts = x;
833}
834
835static void ref_jiffies_delay_section(const int nloops, const int udl, const int ndl)
836{
837 u64 x = 0;
838 int i;
839
840 preempt_disable();
841 for (i = nloops; i >= 0; i--) {
842 x += jiffies;
843 un_delay(udl, ndl);
844 }
845 preempt_enable();
846 stopopts = x;
847}
848
849static const struct ref_scale_ops jiffies_ops = {
850 .readsection = ref_jiffies_section,
851 .delaysection = ref_jiffies_delay_section,
852 .name = "jiffies"
853};
854
855static void ref_preempt_section(const int nloops)
856{
857 int i;
858
859 migrate_disable();
860 for (i = nloops; i >= 0; i--) {
861 preempt_disable();
862 preempt_enable();
863 }
864 migrate_enable();
865}
866
867static void ref_preempt_delay_section(const int nloops, const int udl, const int ndl)
868{
869 int i;
870
871 migrate_disable();
872 for (i = nloops; i >= 0; i--) {
873 preempt_disable();
874 un_delay(udl, ndl);
875 preempt_enable();
876 }
877 migrate_enable();
878}
879
880static const struct ref_scale_ops preempt_ops = {
881 .readsection = ref_preempt_section,
882 .delaysection = ref_preempt_delay_section,
883 .name = "preempt"
884};
885
886static void ref_bh_section(const int nloops)
887{
888 int i;
889
890 preempt_disable();
891 for (i = nloops; i >= 0; i--) {
892 local_bh_disable();
893 local_bh_enable();
894 }
895 preempt_enable();
896}
897
898static void ref_bh_delay_section(const int nloops, const int udl, const int ndl)
899{
900 int i;
901
902 preempt_disable();
903 for (i = nloops; i >= 0; i--) {
904 local_bh_disable();
905 un_delay(udl, ndl);
906 local_bh_enable();
907 }
908 preempt_enable();
909}
910
911static const struct ref_scale_ops bh_ops = {
912 .readsection = ref_bh_section,
913 .delaysection = ref_bh_delay_section,
914 .enable_irqs = true,
915 .name = "bh"
916};
917
918static void ref_irq_section(const int nloops)
919{
920 int i;
921
922 preempt_disable();
923 for (i = nloops; i >= 0; i--) {
924 local_irq_disable();
925 local_irq_enable();
926 }
927 preempt_enable();
928}
929
930static void ref_irq_delay_section(const int nloops, const int udl, const int ndl)
931{
932 int i;
933
934 preempt_disable();
935 for (i = nloops; i >= 0; i--) {
936 local_irq_disable();
937 un_delay(udl, ndl);
938 local_irq_enable();
939 }
940 preempt_enable();
941}
942
943static const struct ref_scale_ops irq_ops = {
944 .readsection = ref_irq_section,
945 .delaysection = ref_irq_delay_section,
946 .name = "irq"
947};
948
949static void ref_irqsave_section(const int nloops)
950{
951 unsigned long flags;
952 int i;
953
954 preempt_disable();
955 for (i = nloops; i >= 0; i--) {
956 local_irq_save(flags);
957 local_irq_restore(flags);
958 }
959 preempt_enable();
960}
961
962static void ref_irqsave_delay_section(const int nloops, const int udl, const int ndl)
963{
964 unsigned long flags;
965 int i;
966
967 preempt_disable();
968 for (i = nloops; i >= 0; i--) {
969 local_irq_save(flags);
970 un_delay(udl, ndl);
971 local_irq_restore(flags);
972 }
973 preempt_enable();
974}
975
976static const struct ref_scale_ops irqsave_ops = {
977 .readsection = ref_irqsave_section,
978 .delaysection = ref_irqsave_delay_section,
979 .name = "irqsave"
980};
981
982////////////////////////////////////////////////////////////////////////
983//
984// Methods leveraging SLAB_TYPESAFE_BY_RCU.
985//
986
987// Item to look up in a typesafe manner. Array of pointers to these.
988struct refscale_typesafe {
989 atomic_t rts_refctr; // Used by all flavors
990 spinlock_t rts_lock;
991 seqlock_t rts_seqlock;
992 unsigned int a;
993 unsigned int b;
994};
995
996static struct kmem_cache *typesafe_kmem_cachep;
997static struct refscale_typesafe **rtsarray;
998static long rtsarray_size;
999static DEFINE_TORTURE_RANDOM_PERCPU(refscale_rand);
1000static bool (*rts_acquire)(struct refscale_typesafe *rtsp, unsigned int *start);
1001static bool (*rts_release)(struct refscale_typesafe *rtsp, unsigned int start);
1002
1003// Conditionally acquire an explicit in-structure reference count.
1004static bool typesafe_ref_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
1005{
1006 return atomic_inc_not_zero(v: &rtsp->rts_refctr);
1007}
1008
1009// Unconditionally release an explicit in-structure reference count.
1010static bool typesafe_ref_release(struct refscale_typesafe *rtsp, unsigned int start)
1011{
1012 if (!atomic_dec_return(v: &rtsp->rts_refctr)) {
1013 WRITE_ONCE(rtsp->a, rtsp->a + 1);
1014 kmem_cache_free(s: typesafe_kmem_cachep, objp: rtsp);
1015 }
1016 return true;
1017}
1018
1019// Unconditionally acquire an explicit in-structure spinlock.
1020static bool typesafe_lock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
1021{
1022 spin_lock(lock: &rtsp->rts_lock);
1023 return true;
1024}
1025
1026// Unconditionally release an explicit in-structure spinlock.
1027static bool typesafe_lock_release(struct refscale_typesafe *rtsp, unsigned int start)
1028{
1029 spin_unlock(lock: &rtsp->rts_lock);
1030 return true;
1031}
1032
1033// Unconditionally acquire an explicit in-structure sequence lock.
1034static bool typesafe_seqlock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
1035{
1036 *start = read_seqbegin(sl: &rtsp->rts_seqlock);
1037 return true;
1038}
1039
1040// Conditionally release an explicit in-structure sequence lock. Return
1041// true if this release was successful, that is, if no retry is required.
1042static bool typesafe_seqlock_release(struct refscale_typesafe *rtsp, unsigned int start)
1043{
1044 return !read_seqretry(sl: &rtsp->rts_seqlock, start);
1045}
1046
1047// Do a read-side critical section with the specified delay in
1048// microseconds and nanoseconds inserted so as to increase probability
1049// of failure.
1050static void typesafe_delay_section(const int nloops, const int udl, const int ndl)
1051{
1052 unsigned int a;
1053 unsigned int b;
1054 int i;
1055 long idx;
1056 struct refscale_typesafe *rtsp;
1057 unsigned int start;
1058
1059 for (i = nloops; i >= 0; i--) {
1060 preempt_disable();
1061 idx = torture_random(this_cpu_ptr(&refscale_rand)) % rtsarray_size;
1062 preempt_enable();
1063retry:
1064 rcu_read_lock();
1065 rtsp = rcu_dereference(rtsarray[idx]);
1066 a = READ_ONCE(rtsp->a);
1067 if (!rts_acquire(rtsp, &start)) {
1068 rcu_read_unlock();
1069 goto retry;
1070 }
1071 if (a != READ_ONCE(rtsp->a)) {
1072 (void)rts_release(rtsp, start);
1073 rcu_read_unlock();
1074 goto retry;
1075 }
1076 un_delay(udl, ndl);
1077 b = READ_ONCE(rtsp->a);
1078 // Remember, seqlock read-side release can fail.
1079 if (!rts_release(rtsp, start)) {
1080 rcu_read_unlock();
1081 goto retry;
1082 }
1083 WARN_ONCE(a != b, "Re-read of ->a changed from %u to %u.\n", a, b);
1084 b = rtsp->b;
1085 rcu_read_unlock();
1086 WARN_ON_ONCE(a * a != b);
1087 }
1088}
1089
1090// Because the acquisition and release methods are expensive, there
1091// is no point in optimizing away the un_delay() function's two checks.
1092// Thus simply define typesafe_read_section() as a simple wrapper around
1093// typesafe_delay_section().
1094static void typesafe_read_section(const int nloops)
1095{
1096 typesafe_delay_section(nloops, udl: 0, ndl: 0);
1097}
1098
1099// Allocate and initialize one refscale_typesafe structure.
1100static struct refscale_typesafe *typesafe_alloc_one(void)
1101{
1102 struct refscale_typesafe *rtsp;
1103
1104 rtsp = kmem_cache_alloc(typesafe_kmem_cachep, GFP_KERNEL);
1105 if (!rtsp)
1106 return NULL;
1107 atomic_set(v: &rtsp->rts_refctr, i: 1);
1108 WRITE_ONCE(rtsp->a, rtsp->a + 1);
1109 WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a);
1110 return rtsp;
1111}
1112
1113// Slab-allocator constructor for refscale_typesafe structures created
1114// out of a new slab of system memory.
1115static void refscale_typesafe_ctor(void *rtsp_in)
1116{
1117 struct refscale_typesafe *rtsp = rtsp_in;
1118
1119 spin_lock_init(&rtsp->rts_lock);
1120 seqlock_init(&rtsp->rts_seqlock);
1121 preempt_disable();
1122 rtsp->a = torture_random(this_cpu_ptr(&refscale_rand));
1123 preempt_enable();
1124}
1125
1126static const struct ref_scale_ops typesafe_ref_ops;
1127static const struct ref_scale_ops typesafe_lock_ops;
1128static const struct ref_scale_ops typesafe_seqlock_ops;
1129
1130// Initialize for a typesafe test.
1131static bool typesafe_init(void)
1132{
1133 long idx;
1134 long si = lookup_instances;
1135
1136 typesafe_kmem_cachep = kmem_cache_create("refscale_typesafe",
1137 sizeof(struct refscale_typesafe), sizeof(void *),
1138 SLAB_TYPESAFE_BY_RCU, refscale_typesafe_ctor);
1139 if (!typesafe_kmem_cachep)
1140 return false;
1141 if (si < 0)
1142 si = -si * nr_cpu_ids;
1143 else if (si == 0)
1144 si = nr_cpu_ids;
1145 rtsarray_size = si;
1146 rtsarray = kcalloc(si, sizeof(*rtsarray), GFP_KERNEL);
1147 if (!rtsarray)
1148 return false;
1149 for (idx = 0; idx < rtsarray_size; idx++) {
1150 rtsarray[idx] = typesafe_alloc_one();
1151 if (!rtsarray[idx])
1152 return false;
1153 }
1154 if (cur_ops == &typesafe_ref_ops) {
1155 rts_acquire = typesafe_ref_acquire;
1156 rts_release = typesafe_ref_release;
1157 } else if (cur_ops == &typesafe_lock_ops) {
1158 rts_acquire = typesafe_lock_acquire;
1159 rts_release = typesafe_lock_release;
1160 } else if (cur_ops == &typesafe_seqlock_ops) {
1161 rts_acquire = typesafe_seqlock_acquire;
1162 rts_release = typesafe_seqlock_release;
1163 } else {
1164 WARN_ON_ONCE(1);
1165 return false;
1166 }
1167 return true;
1168}
1169
1170// Clean up after a typesafe test.
1171static void typesafe_cleanup(void)
1172{
1173 long idx;
1174
1175 if (rtsarray) {
1176 for (idx = 0; idx < rtsarray_size; idx++)
1177 kmem_cache_free(s: typesafe_kmem_cachep, objp: rtsarray[idx]);
1178 kfree(objp: rtsarray);
1179 rtsarray = NULL;
1180 rtsarray_size = 0;
1181 }
1182 kmem_cache_destroy(s: typesafe_kmem_cachep);
1183 typesafe_kmem_cachep = NULL;
1184 rts_acquire = NULL;
1185 rts_release = NULL;
1186}
1187
1188// The typesafe_init() function distinguishes these structures by address.
1189static const struct ref_scale_ops typesafe_ref_ops = {
1190 .init = typesafe_init,
1191 .cleanup = typesafe_cleanup,
1192 .readsection = typesafe_read_section,
1193 .delaysection = typesafe_delay_section,
1194 .name = "typesafe_ref"
1195};
1196
1197static const struct ref_scale_ops typesafe_lock_ops = {
1198 .init = typesafe_init,
1199 .cleanup = typesafe_cleanup,
1200 .readsection = typesafe_read_section,
1201 .delaysection = typesafe_delay_section,
1202 .name = "typesafe_lock"
1203};
1204
1205static const struct ref_scale_ops typesafe_seqlock_ops = {
1206 .init = typesafe_init,
1207 .cleanup = typesafe_cleanup,
1208 .readsection = typesafe_read_section,
1209 .delaysection = typesafe_delay_section,
1210 .name = "typesafe_seqlock"
1211};
1212
1213static void rcu_scale_one_reader(void)
1214{
1215 if (readdelay <= 0)
1216 cur_ops->readsection(loops);
1217 else
1218 cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);
1219}
1220
1221// Warm up cache, or, if needed run a series of rcu_scale_one_reader()
1222// to allow multiple rcuscale guest OSes to collect mutually valid data.
1223static void rcu_scale_warm_cool(void)
1224{
1225 unsigned long jdone = jiffies + (guest_os_delay > 0 ? guest_os_delay * HZ : -1);
1226
1227 do {
1228 rcu_scale_one_reader();
1229 cond_resched();
1230 } while (time_before(jiffies, jdone));
1231}
1232
1233// Reader kthread. Repeatedly does empty RCU read-side
1234// critical section, minimizing update-side interference.
1235static int
1236ref_scale_reader(void *arg)
1237{
1238 unsigned long flags;
1239 long me = (long)arg;
1240 struct reader_task *rt = &(reader_tasks[me]);
1241 u64 start;
1242 s64 duration;
1243
1244 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me);
1245 WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)));
1246 set_user_nice(current, MAX_NICE);
1247 atomic_inc(v: &n_init);
1248 if (holdoff)
1249 schedule_timeout_interruptible(timeout: holdoff * HZ);
1250repeat:
1251 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, raw_smp_processor_id());
1252
1253 // Wait for signal that this reader can start.
1254 wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||
1255 torture_must_stop());
1256
1257 if (torture_must_stop())
1258 goto end;
1259
1260 // Make sure that the CPU is affinitized appropriately during testing.
1261 WARN_ON_ONCE(raw_smp_processor_id() != me % nr_cpu_ids);
1262
1263 WRITE_ONCE(rt->start_reader, 0);
1264 if (!atomic_dec_return(v: &n_started))
1265 while (atomic_read_acquire(v: &n_started))
1266 cpu_relax();
1267
1268 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx);
1269
1270
1271 // To reduce noise, do an initial cache-warming invocation, check
1272 // in, and then keep warming until everyone has checked in.
1273 rcu_scale_one_reader();
1274 if (!atomic_dec_return(v: &n_warmedup))
1275 while (atomic_read_acquire(v: &n_warmedup))
1276 rcu_scale_one_reader();
1277 // Also keep interrupts disabled when it is safe to do so, which
1278 // it is not for local_bh_enable(). This also has the effect of
1279 // preventing entries into slow path for rcu_read_unlock().
1280 if (!cur_ops->enable_irqs)
1281 local_irq_save(flags);
1282 start = ktime_get_mono_fast_ns();
1283
1284 rcu_scale_one_reader();
1285
1286 duration = ktime_get_mono_fast_ns() - start;
1287 if (!cur_ops->enable_irqs)
1288 local_irq_restore(flags);
1289
1290 rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;
1291 // To reduce runtime-skew noise, do maintain-load invocations until
1292 // everyone is done.
1293 if (!atomic_dec_return(v: &n_cooleddown))
1294 while (atomic_read_acquire(v: &n_cooleddown))
1295 rcu_scale_one_reader();
1296
1297 if (atomic_dec_and_test(v: &nreaders_exp))
1298 wake_up(&main_wq);
1299
1300 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)",
1301 me, exp_idx, atomic_read(&nreaders_exp));
1302
1303 if (!torture_must_stop())
1304 goto repeat;
1305end:
1306 torture_kthread_stopping(title: "ref_scale_reader");
1307 return 0;
1308}
1309
1310static void reset_readers(void)
1311{
1312 int i;
1313 struct reader_task *rt;
1314
1315 for (i = 0; i < nreaders; i++) {
1316 rt = &(reader_tasks[i]);
1317
1318 rt->last_duration_ns = 0;
1319 }
1320}
1321
1322// Print the results of each reader and return the sum of all their durations.
1323static u64 process_durations(int n)
1324{
1325 int i;
1326 struct reader_task *rt;
1327 struct seq_buf s;
1328 char *buf;
1329 u64 sum = 0;
1330
1331 buf = kmalloc(800 + 64, GFP_KERNEL);
1332 if (!buf)
1333 return 0;
1334 seq_buf_init(s: &s, buf, size: 800 + 64);
1335
1336 seq_buf_printf(s: &s, fmt: "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",
1337 exp_idx);
1338
1339 for (i = 0; i < n && !torture_must_stop(); i++) {
1340 rt = &(reader_tasks[i]);
1341
1342 if (i % 5 == 0)
1343 seq_buf_putc(s: &s, c: '\n');
1344
1345 if (seq_buf_used(s: &s) >= 800) {
1346 pr_alert("%s", seq_buf_str(&s));
1347 seq_buf_clear(s: &s);
1348 }
1349
1350 seq_buf_printf(s: &s, fmt: "%d: %llu\t", i, rt->last_duration_ns);
1351
1352 sum += rt->last_duration_ns;
1353 }
1354 pr_alert("%s\n", seq_buf_str(&s));
1355
1356 kfree(objp: buf);
1357 return sum;
1358}
1359
1360// The main_func is the main orchestrator, it performs a bunch of
1361// experiments. For every experiment, it orders all the readers
1362// involved to start and waits for them to finish the experiment. It
1363// then reads their timestamps and starts the next experiment. Each
1364// experiment progresses from 1 concurrent reader to N of them at which
1365// point all the timestamps are printed.
1366static int main_func(void *arg)
1367{
1368 int exp, r;
1369 char buf1[64];
1370 char *buf;
1371 u64 *result_avg;
1372
1373 set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids));
1374 set_user_nice(current, MAX_NICE);
1375
1376 VERBOSE_SCALEOUT("main_func task started");
1377 result_avg = kcalloc(nruns, sizeof(*result_avg), GFP_KERNEL);
1378 buf = kzalloc(800 + 64, GFP_KERNEL);
1379 if (!result_avg || !buf) {
1380 SCALEOUT_ERRSTRING("out of memory");
1381 goto oom_exit;
1382 }
1383 if (holdoff)
1384 schedule_timeout_interruptible(timeout: holdoff * HZ);
1385
1386 // Wait for all threads to start.
1387 atomic_inc(v: &n_init);
1388 while (atomic_read(v: &n_init) < nreaders + 1)
1389 schedule_timeout_uninterruptible(timeout: 1);
1390
1391 // Start exp readers up per experiment
1392 rcu_scale_warm_cool();
1393 for (exp = 0; exp < nruns && !torture_must_stop(); exp++) {
1394 if (torture_must_stop())
1395 goto end;
1396
1397 reset_readers();
1398 atomic_set(v: &nreaders_exp, i: nreaders);
1399 atomic_set(v: &n_started, i: nreaders);
1400 atomic_set(v: &n_warmedup, i: nreaders);
1401 atomic_set(v: &n_cooleddown, i: nreaders);
1402
1403 exp_idx = exp;
1404
1405 for (r = 0; r < nreaders; r++) {
1406 smp_store_release(&reader_tasks[r].start_reader, 1);
1407 wake_up(&reader_tasks[r].wq);
1408 }
1409
1410 VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers",
1411 nreaders);
1412
1413 wait_event(main_wq,
1414 !atomic_read(&nreaders_exp) || torture_must_stop());
1415
1416 VERBOSE_SCALEOUT("main_func: experiment ended");
1417
1418 if (torture_must_stop())
1419 goto end;
1420
1421 result_avg[exp] = div_u64(dividend: 1000 * process_durations(n: nreaders), divisor: nreaders * loops);
1422 }
1423 rcu_scale_warm_cool();
1424
1425 // Print the average of all experiments
1426 SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");
1427
1428 pr_alert("Runs\tTime(ns)\n");
1429 for (exp = 0; exp < nruns; exp++) {
1430 u64 avg;
1431 u32 rem;
1432
1433 avg = div_u64_rem(dividend: result_avg[exp], divisor: 1000, remainder: &rem);
1434 sprintf(buf: buf1, fmt: "%d\t%llu.%03u\n", exp + 1, avg, rem);
1435 strcat(p: buf, q: buf1);
1436 if (strlen(buf) >= 800) {
1437 pr_alert("%s", buf);
1438 buf[0] = 0;
1439 }
1440 }
1441
1442 pr_alert("%s", buf);
1443
1444oom_exit:
1445 // This will shutdown everything including us.
1446 if (shutdown) {
1447 shutdown_start = 1;
1448 wake_up(&shutdown_wq);
1449 }
1450
1451 // Wait for torture to stop us
1452 while (!torture_must_stop())
1453 schedule_timeout_uninterruptible(timeout: 1);
1454
1455end:
1456 torture_kthread_stopping(title: "main_func");
1457 kfree(objp: result_avg);
1458 kfree(objp: buf);
1459 return 0;
1460}
1461
1462static void
1463ref_scale_print_module_parms(const struct ref_scale_ops *cur_ops, const char *tag)
1464{
1465 pr_alert("%s" SCALE_FLAG
1466 "--- %s: verbose=%d verbose_batched=%d shutdown=%d holdoff=%d lookup_instances=%ld loops=%d nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag,
1467 verbose, verbose_batched, shutdown, holdoff, lookup_instances, loops, nreaders, nruns, readdelay);
1468}
1469
1470static void
1471ref_scale_cleanup(void)
1472{
1473 int i;
1474
1475 if (torture_cleanup_begin())
1476 return;
1477
1478 if (!cur_ops) {
1479 torture_cleanup_end();
1480 return;
1481 }
1482
1483 if (reader_tasks) {
1484 for (i = 0; i < nreaders; i++)
1485 torture_stop_kthread("ref_scale_reader",
1486 reader_tasks[i].task);
1487 }
1488 kfree(objp: reader_tasks);
1489 reader_tasks = NULL;
1490
1491 torture_stop_kthread("main_task", main_task);
1492
1493 // Do scale-type-specific cleanup operations.
1494 if (cur_ops->cleanup != NULL)
1495 cur_ops->cleanup();
1496
1497 torture_cleanup_end();
1498}
1499
1500// Shutdown kthread. Just waits to be awakened, then shuts down system.
1501static int
1502ref_scale_shutdown(void *arg)
1503{
1504 wait_event_idle(shutdown_wq, shutdown_start);
1505
1506 smp_mb(); // Wake before output.
1507 ref_scale_cleanup();
1508 kernel_power_off();
1509
1510 return -EINVAL;
1511}
1512
1513static int __init
1514ref_scale_init(void)
1515{
1516 long i;
1517 int firsterr = 0;
1518 static const struct ref_scale_ops *scale_ops[] = {
1519 &rcu_ops, &srcu_ops, &srcu_fast_ops, &srcu_fast_updown_ops,
1520 RCU_TRACE_OPS RCU_TASKS_OPS
1521 &refcnt_ops, &percpuinc_ops, &incpercpu_ops, &incpercpupreempt_ops,
1522 &incpercpubh_ops, &incpercpuirqsave_ops,
1523 &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops,
1524 &sched_clock_ops, &clock_ops, &jiffies_ops,
1525 &preempt_ops, &bh_ops, &irq_ops, &irqsave_ops,
1526 &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
1527 };
1528
1529 if (!torture_init_begin(ttype: scale_type, v: verbose))
1530 return -EBUSY;
1531
1532 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
1533 cur_ops = scale_ops[i];
1534 if (strcmp(scale_type, cur_ops->name) == 0)
1535 break;
1536 }
1537 if (i == ARRAY_SIZE(scale_ops)) {
1538 pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
1539 pr_alert("rcu-scale types:");
1540 for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
1541 pr_cont(" %s", scale_ops[i]->name);
1542 pr_cont("\n");
1543 firsterr = -EINVAL;
1544 cur_ops = NULL;
1545 goto unwind;
1546 }
1547 if (cur_ops->init)
1548 if (!cur_ops->init()) {
1549 firsterr = -EUCLEAN;
1550 goto unwind;
1551 }
1552
1553 ref_scale_print_module_parms(cur_ops, tag: "Start of test");
1554
1555 // Shutdown task
1556 if (shutdown) {
1557 init_waitqueue_head(&shutdown_wq);
1558 firsterr = torture_create_kthread(ref_scale_shutdown, NULL,
1559 shutdown_task);
1560 if (torture_init_error(firsterr))
1561 goto unwind;
1562 schedule_timeout_uninterruptible(timeout: 1);
1563 }
1564
1565 // Reader tasks (default to ~75% of online CPUs).
1566 if (nreaders < 0)
1567 nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2);
1568 if (WARN_ONCE(loops <= 0, "%s: loops = %d, adjusted to 1\n", __func__, loops))
1569 loops = 1;
1570 if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders))
1571 nreaders = 1;
1572 if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns))
1573 nruns = 1;
1574 if (WARN_ONCE(loops > INT_MAX / nreaders,
1575 "%s: nreaders * loops will overflow, adjusted loops to %d",
1576 __func__, INT_MAX / nreaders))
1577 loops = INT_MAX / nreaders;
1578 reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]),
1579 GFP_KERNEL);
1580 if (!reader_tasks) {
1581 SCALEOUT_ERRSTRING("out of memory");
1582 firsterr = -ENOMEM;
1583 goto unwind;
1584 }
1585
1586 VERBOSE_SCALEOUT("Starting %d reader threads", nreaders);
1587
1588 for (i = 0; i < nreaders; i++) {
1589 init_waitqueue_head(&reader_tasks[i].wq);
1590 firsterr = torture_create_kthread(ref_scale_reader, (void *)i,
1591 reader_tasks[i].task);
1592 if (torture_init_error(firsterr))
1593 goto unwind;
1594 }
1595
1596 // Main Task
1597 init_waitqueue_head(&main_wq);
1598 firsterr = torture_create_kthread(main_func, NULL, main_task);
1599 if (torture_init_error(firsterr))
1600 goto unwind;
1601
1602 torture_init_end();
1603 return 0;
1604
1605unwind:
1606 torture_init_end();
1607 ref_scale_cleanup();
1608 if (shutdown) {
1609 WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST));
1610 kernel_power_off();
1611 }
1612 return firsterr;
1613}
1614
1615module_init(ref_scale_init);
1616module_exit(ref_scale_cleanup);
1617

source code of linux/kernel/rcu/refscale.c