forked from torvalds/linux
-
Notifications
You must be signed in to change notification settings - Fork 144
Expand file tree
/
Copy pathpercpu_freelist.c
More file actions
137 lines (117 loc) · 3.05 KB
/
percpu_freelist.c
File metadata and controls
137 lines (117 loc) · 3.05 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2016 Facebook
*/
#include "percpu_freelist.h"
int pcpu_freelist_init(struct pcpu_freelist *s)
{
int cpu;
s->freelist = alloc_percpu(struct pcpu_freelist_head);
if (!s->freelist)
return -ENOMEM;
for_each_possible_cpu(cpu) {
struct pcpu_freelist_head *head = per_cpu_ptr(s->freelist, cpu);
raw_res_spin_lock_init(&head->lock);
head->first = NULL;
}
return 0;
}
void pcpu_freelist_destroy(struct pcpu_freelist *s)
{
free_percpu(s->freelist);
}
static inline void pcpu_freelist_push_node(struct pcpu_freelist_head *head,
struct pcpu_freelist_node *node)
{
node->next = head->first;
WRITE_ONCE(head->first, node);
}
static inline bool ___pcpu_freelist_push(struct pcpu_freelist_head *head,
struct pcpu_freelist_node *node)
{
if (raw_res_spin_lock(&head->lock))
return false;
pcpu_freelist_push_node(head, node);
raw_res_spin_unlock(&head->lock);
return true;
}
void __pcpu_freelist_push(struct pcpu_freelist *s,
struct pcpu_freelist_node *node)
{
struct pcpu_freelist_head *head;
int cpu;
if (___pcpu_freelist_push(this_cpu_ptr(s->freelist), node))
return;
while (true) {
for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) {
if (cpu == raw_smp_processor_id())
continue;
head = per_cpu_ptr(s->freelist, cpu);
if (raw_res_spin_lock(&head->lock))
continue;
pcpu_freelist_push_node(head, node);
raw_res_spin_unlock(&head->lock);
return;
}
}
}
void pcpu_freelist_push(struct pcpu_freelist *s,
struct pcpu_freelist_node *node)
{
unsigned long flags;
local_irq_save(flags);
__pcpu_freelist_push(s, node);
local_irq_restore(flags);
}
void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size,
u32 nr_elems)
{
struct pcpu_freelist_head *head;
unsigned int cpu, cpu_idx, i, j, n, m;
n = nr_elems / num_possible_cpus();
m = nr_elems % num_possible_cpus();
cpu_idx = 0;
for_each_possible_cpu(cpu) {
head = per_cpu_ptr(s->freelist, cpu);
j = n + (cpu_idx < m ? 1 : 0);
for (i = 0; i < j; i++) {
/* No locking required as this is not visible yet. */
pcpu_freelist_push_node(head, buf);
buf += elem_size;
}
cpu_idx++;
}
}
static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s)
{
struct pcpu_freelist_node *node = NULL;
struct pcpu_freelist_head *head;
int cpu;
for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) {
head = per_cpu_ptr(s->freelist, cpu);
if (!READ_ONCE(head->first))
continue;
if (raw_res_spin_lock(&head->lock))
continue;
node = head->first;
if (node) {
WRITE_ONCE(head->first, node->next);
raw_res_spin_unlock(&head->lock);
return node;
}
raw_res_spin_unlock(&head->lock);
}
return node;
}
struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *s)
{
return ___pcpu_freelist_pop(s);
}
struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s)
{
struct pcpu_freelist_node *ret;
unsigned long flags;
local_irq_save(flags);
ret = __pcpu_freelist_pop(s);
local_irq_restore(flags);
return ret;
}