1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3
4#include <linux/slab.h>
5#include <linux/dma-mapping.h>
6
7#include "lima_device.h"
8#include "lima_vm.h"
9#include "lima_gem.h"
10#include "lima_regs.h"
11
12struct lima_bo_va {
13 struct list_head list;
14 unsigned int ref_count;
15
16 struct drm_mm_node node;
17
18 struct lima_vm *vm;
19};
20
21#define LIMA_VM_PD_SHIFT 22
22#define LIMA_VM_PT_SHIFT 12
23#define LIMA_VM_PB_SHIFT (LIMA_VM_PD_SHIFT + LIMA_VM_NUM_PT_PER_BT_SHIFT)
24#define LIMA_VM_BT_SHIFT LIMA_VM_PT_SHIFT
25
26#define LIMA_VM_PT_MASK ((1 << LIMA_VM_PD_SHIFT) - 1)
27#define LIMA_VM_BT_MASK ((1 << LIMA_VM_PB_SHIFT) - 1)
28
29#define LIMA_PDE(va) (va >> LIMA_VM_PD_SHIFT)
30#define LIMA_PTE(va) ((va & LIMA_VM_PT_MASK) >> LIMA_VM_PT_SHIFT)
31#define LIMA_PBE(va) (va >> LIMA_VM_PB_SHIFT)
32#define LIMA_BTE(va) ((va & LIMA_VM_BT_MASK) >> LIMA_VM_BT_SHIFT)
33
34
35static void lima_vm_unmap_range(struct lima_vm *vm, u32 start, u32 end)
36{
37 u32 addr;
38
39 for (addr = start; addr <= end; addr += LIMA_PAGE_SIZE) {
40 u32 pbe = LIMA_PBE(addr);
41 u32 bte = LIMA_BTE(addr);
42
43 vm->bts[pbe].cpu[bte] = 0;
44 }
45}
46
47static int lima_vm_map_page(struct lima_vm *vm, dma_addr_t pa, u32 va)
48{
49 u32 pbe = LIMA_PBE(va);
50 u32 bte = LIMA_BTE(va);
51
52 if (!vm->bts[pbe].cpu) {
53 dma_addr_t pts;
54 u32 *pd;
55 int j;
56
57 vm->bts[pbe].cpu = dma_alloc_wc(
58 dev: vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,
59 dma_addr: &vm->bts[pbe].dma, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
60 if (!vm->bts[pbe].cpu)
61 return -ENOMEM;
62
63 pts = vm->bts[pbe].dma;
64 pd = vm->pd.cpu + (pbe << LIMA_VM_NUM_PT_PER_BT_SHIFT);
65 for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) {
66 pd[j] = pts | LIMA_VM_FLAG_PRESENT;
67 pts += LIMA_PAGE_SIZE;
68 }
69 }
70
71 vm->bts[pbe].cpu[bte] = pa | LIMA_VM_FLAGS_CACHE;
72
73 return 0;
74}
75
76static struct lima_bo_va *
77lima_vm_bo_find(struct lima_vm *vm, struct lima_bo *bo)
78{
79 struct lima_bo_va *bo_va, *ret = NULL;
80
81 list_for_each_entry(bo_va, &bo->va, list) {
82 if (bo_va->vm == vm) {
83 ret = bo_va;
84 break;
85 }
86 }
87
88 return ret;
89}
90
91int lima_vm_bo_add(struct lima_vm *vm, struct lima_bo *bo, bool create)
92{
93 struct lima_bo_va *bo_va;
94 struct sg_dma_page_iter sg_iter;
95 int offset = 0, err;
96
97 mutex_lock(&bo->lock);
98
99 bo_va = lima_vm_bo_find(vm, bo);
100 if (bo_va) {
101 bo_va->ref_count++;
102 mutex_unlock(lock: &bo->lock);
103 return 0;
104 }
105
106 /* should not create new bo_va if not asked by caller */
107 if (!create) {
108 mutex_unlock(lock: &bo->lock);
109 return -ENOENT;
110 }
111
112 bo_va = kzalloc(sizeof(*bo_va), GFP_KERNEL);
113 if (!bo_va) {
114 err = -ENOMEM;
115 goto err_out0;
116 }
117
118 bo_va->vm = vm;
119 bo_va->ref_count = 1;
120
121 mutex_lock(&vm->lock);
122
123 err = drm_mm_insert_node(mm: &vm->mm, node: &bo_va->node, size: lima_bo_size(bo));
124 if (err)
125 goto err_out1;
126
127 for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, 0) {
128 err = lima_vm_map_page(vm, pa: sg_page_iter_dma_address(dma_iter: &sg_iter),
129 va: bo_va->node.start + offset);
130 if (err)
131 goto err_out2;
132
133 offset += PAGE_SIZE;
134 }
135
136 mutex_unlock(lock: &vm->lock);
137
138 list_add_tail(new: &bo_va->list, head: &bo->va);
139
140 mutex_unlock(lock: &bo->lock);
141 return 0;
142
143err_out2:
144 if (offset)
145 lima_vm_unmap_range(vm, start: bo_va->node.start, end: bo_va->node.start + offset - 1);
146 drm_mm_remove_node(node: &bo_va->node);
147err_out1:
148 mutex_unlock(lock: &vm->lock);
149 kfree(objp: bo_va);
150err_out0:
151 mutex_unlock(lock: &bo->lock);
152 return err;
153}
154
155void lima_vm_bo_del(struct lima_vm *vm, struct lima_bo *bo)
156{
157 struct lima_bo_va *bo_va;
158 u32 size;
159
160 mutex_lock(&bo->lock);
161
162 bo_va = lima_vm_bo_find(vm, bo);
163 if (--bo_va->ref_count > 0) {
164 mutex_unlock(lock: &bo->lock);
165 return;
166 }
167
168 mutex_lock(&vm->lock);
169
170 size = bo->heap_size ? bo->heap_size : bo_va->node.size;
171 lima_vm_unmap_range(vm, start: bo_va->node.start,
172 end: bo_va->node.start + size - 1);
173
174 drm_mm_remove_node(node: &bo_va->node);
175
176 mutex_unlock(lock: &vm->lock);
177
178 list_del(entry: &bo_va->list);
179
180 mutex_unlock(lock: &bo->lock);
181
182 kfree(objp: bo_va);
183}
184
185u32 lima_vm_get_va(struct lima_vm *vm, struct lima_bo *bo)
186{
187 struct lima_bo_va *bo_va;
188 u32 ret;
189
190 mutex_lock(&bo->lock);
191
192 bo_va = lima_vm_bo_find(vm, bo);
193 ret = bo_va->node.start;
194
195 mutex_unlock(lock: &bo->lock);
196
197 return ret;
198}
199
200struct lima_vm *lima_vm_create(struct lima_device *dev)
201{
202 struct lima_vm *vm;
203
204 vm = kzalloc(sizeof(*vm), GFP_KERNEL);
205 if (!vm)
206 return NULL;
207
208 vm->dev = dev;
209 mutex_init(&vm->lock);
210 kref_init(kref: &vm->refcount);
211
212 vm->pd.cpu = dma_alloc_wc(dev: dev->dev, LIMA_PAGE_SIZE, dma_addr: &vm->pd.dma,
213 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
214 if (!vm->pd.cpu)
215 goto err_out0;
216
217 if (dev->dlbu_cpu) {
218 int err = lima_vm_map_page(
219 vm, pa: dev->dlbu_dma, LIMA_VA_RESERVE_DLBU);
220 if (err)
221 goto err_out1;
222 }
223
224 drm_mm_init(mm: &vm->mm, start: dev->va_start, size: dev->va_end - dev->va_start);
225
226 return vm;
227
228err_out1:
229 dma_free_wc(dev: dev->dev, LIMA_PAGE_SIZE, cpu_addr: vm->pd.cpu, dma_addr: vm->pd.dma);
230err_out0:
231 kfree(objp: vm);
232 return NULL;
233}
234
235void lima_vm_release(struct kref *kref)
236{
237 struct lima_vm *vm = container_of(kref, struct lima_vm, refcount);
238 int i;
239
240 drm_mm_takedown(mm: &vm->mm);
241
242 for (i = 0; i < LIMA_VM_NUM_BT; i++) {
243 if (vm->bts[i].cpu)
244 dma_free_wc(dev: vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,
245 cpu_addr: vm->bts[i].cpu, dma_addr: vm->bts[i].dma);
246 }
247
248 if (vm->pd.cpu)
249 dma_free_wc(dev: vm->dev->dev, LIMA_PAGE_SIZE, cpu_addr: vm->pd.cpu, dma_addr: vm->pd.dma);
250
251 kfree(objp: vm);
252}
253
254void lima_vm_print(struct lima_vm *vm)
255{
256 int i, j, k;
257 u32 *pd, *pt;
258
259 if (!vm->pd.cpu)
260 return;
261
262 pd = vm->pd.cpu;
263 for (i = 0; i < LIMA_VM_NUM_BT; i++) {
264 if (!vm->bts[i].cpu)
265 continue;
266
267 pt = vm->bts[i].cpu;
268 for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) {
269 int idx = (i << LIMA_VM_NUM_PT_PER_BT_SHIFT) + j;
270
271 printk(KERN_INFO "lima vm pd %03x:%08x\n", idx, pd[idx]);
272
273 for (k = 0; k < LIMA_PAGE_ENT_NUM; k++) {
274 u32 pte = *pt++;
275
276 if (pte)
277 printk(KERN_INFO " pt %03x:%08x\n", k, pte);
278 }
279 }
280 }
281}
282
283int lima_vm_map_bo(struct lima_vm *vm, struct lima_bo *bo, int pageoff)
284{
285 struct lima_bo_va *bo_va;
286 struct sg_dma_page_iter sg_iter;
287 int offset = 0, err;
288 u32 base;
289
290 mutex_lock(&bo->lock);
291
292 bo_va = lima_vm_bo_find(vm, bo);
293 if (!bo_va) {
294 err = -ENOENT;
295 goto err_out0;
296 }
297
298 mutex_lock(&vm->lock);
299
300 base = bo_va->node.start + (pageoff << PAGE_SHIFT);
301 for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, pageoff) {
302 err = lima_vm_map_page(vm, pa: sg_page_iter_dma_address(dma_iter: &sg_iter),
303 va: base + offset);
304 if (err)
305 goto err_out1;
306
307 offset += PAGE_SIZE;
308 }
309
310 mutex_unlock(lock: &vm->lock);
311
312 mutex_unlock(lock: &bo->lock);
313 return 0;
314
315err_out1:
316 if (offset)
317 lima_vm_unmap_range(vm, start: base, end: base + offset - 1);
318 mutex_unlock(lock: &vm->lock);
319err_out0:
320 mutex_unlock(lock: &bo->lock);
321 return err;
322}
323

source code of linux/drivers/gpu/drm/lima/lima_vm.c