1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/arch/alpha/kernel/pci_iommu.c
4 */
5
6#include <linux/kernel.h>
7#include <linux/mm.h>
8#include <linux/pci.h>
9#include <linux/gfp.h>
10#include <linux/memblock.h>
11#include <linux/export.h>
12#include <linux/scatterlist.h>
13#include <linux/log2.h>
14#include <linux/dma-map-ops.h>
15#include <linux/iommu-helper.h>
16#include <linux/string_choices.h>
17
18#include <asm/io.h>
19#include <asm/hwrpb.h>
20
21#include "proto.h"
22#include "pci_impl.h"
23
24
25#define DEBUG_ALLOC 0
26#if DEBUG_ALLOC > 0
27# define DBGA(args...) printk(KERN_DEBUG args)
28#else
29# define DBGA(args...)
30#endif
31#if DEBUG_ALLOC > 1
32# define DBGA2(args...) printk(KERN_DEBUG args)
33#else
34# define DBGA2(args...)
35#endif
36
37#define DEBUG_NODIRECT 0
38
39#define ISA_DMA_MASK 0x00ffffff
40
41static inline unsigned long
42mk_iommu_pte(unsigned long paddr)
43{
44 return (paddr >> (PAGE_SHIFT-1)) | 1;
45}
46
47/* Return the minimum of MAX or the first power of two larger
48 than main memory. */
49
50unsigned long
51size_for_memory(unsigned long max)
52{
53 unsigned long mem = max_low_pfn << PAGE_SHIFT;
54 if (mem < max)
55 max = roundup_pow_of_two(mem);
56 return max;
57}
58
59struct pci_iommu_arena * __init
60iommu_arena_new_node(int nid, struct pci_controller *hose, dma_addr_t base,
61 unsigned long window_size, unsigned long align)
62{
63 unsigned long mem_size;
64 struct pci_iommu_arena *arena;
65
66 mem_size = window_size / (PAGE_SIZE / sizeof(unsigned long));
67
68 /* Note that the TLB lookup logic uses bitwise concatenation,
69 not addition, so the required arena alignment is based on
70 the size of the window. Retain the align parameter so that
71 particular systems can over-align the arena. */
72 if (align < mem_size)
73 align = mem_size;
74
75 arena = memblock_alloc_or_panic(sizeof(*arena), SMP_CACHE_BYTES);
76 arena->ptes = memblock_alloc_or_panic(mem_size, align);
77
78 spin_lock_init(&arena->lock);
79 arena->hose = hose;
80 arena->dma_base = base;
81 arena->size = window_size;
82 arena->next_entry = 0;
83
84 /* Align allocations to a multiple of a page size. Not needed
85 unless there are chip bugs. */
86 arena->align_entry = 1;
87
88 return arena;
89}
90
91struct pci_iommu_arena * __init
92iommu_arena_new(struct pci_controller *hose, dma_addr_t base,
93 unsigned long window_size, unsigned long align)
94{
95 return iommu_arena_new_node(nid: 0, hose, base, window_size, align);
96}
97
98/* Must be called with the arena lock held */
99static long
100iommu_arena_find_pages(struct device *dev, struct pci_iommu_arena *arena,
101 long n, long mask)
102{
103 unsigned long *ptes;
104 long i, p, nent;
105 int pass = 0;
106 unsigned long base;
107 unsigned long boundary_size;
108
109 base = arena->dma_base >> PAGE_SHIFT;
110 boundary_size = dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT);
111
112 /* Search forward for the first mask-aligned sequence of N free ptes */
113 ptes = arena->ptes;
114 nent = arena->size >> PAGE_SHIFT;
115 p = ALIGN(arena->next_entry, mask + 1);
116 i = 0;
117
118again:
119 while (i < n && p+i < nent) {
120 if (!i && iommu_is_span_boundary(index: p, nr: n, shift: base, boundary_size)) {
121 p = ALIGN(p + 1, mask + 1);
122 goto again;
123 }
124
125 if (ptes[p+i]) {
126 p = ALIGN(p + i + 1, mask + 1);
127 i = 0;
128 } else {
129 i = i + 1;
130 }
131 }
132
133 if (i < n) {
134 if (pass < 1) {
135 /*
136 * Reached the end. Flush the TLB and restart
137 * the search from the beginning.
138 */
139 alpha_mv.mv_pci_tbi(arena->hose, 0, -1);
140
141 pass++;
142 p = 0;
143 i = 0;
144 goto again;
145 } else
146 return -1;
147 }
148
149 /* Success. It's the responsibility of the caller to mark them
150 in use before releasing the lock */
151 return p;
152}
153
154static long
155iommu_arena_alloc(struct device *dev, struct pci_iommu_arena *arena, long n,
156 unsigned int align)
157{
158 unsigned long flags;
159 unsigned long *ptes;
160 long i, p, mask;
161
162 spin_lock_irqsave(&arena->lock, flags);
163
164 /* Search for N empty ptes */
165 ptes = arena->ptes;
166 mask = max(align, arena->align_entry) - 1;
167 p = iommu_arena_find_pages(dev, arena, n, mask);
168 if (p < 0) {
169 spin_unlock_irqrestore(lock: &arena->lock, flags);
170 return -1;
171 }
172
173 /* Success. Mark them all in use, ie not zero and invalid
174 for the iommu tlb that could load them from under us.
175 The chip specific bits will fill this in with something
176 kosher when we return. */
177 for (i = 0; i < n; ++i)
178 ptes[p+i] = IOMMU_INVALID_PTE;
179
180 arena->next_entry = p + n;
181 spin_unlock_irqrestore(lock: &arena->lock, flags);
182
183 return p;
184}
185
186static void
187iommu_arena_free(struct pci_iommu_arena *arena, long ofs, long n)
188{
189 unsigned long *p;
190 long i;
191
192 p = arena->ptes + ofs;
193 for (i = 0; i < n; ++i)
194 p[i] = 0;
195}
196
197/*
198 * True if the machine supports DAC addressing, and DEV can
199 * make use of it given MASK.
200 */
201static int pci_dac_dma_supported(struct pci_dev *dev, u64 mask)
202{
203 dma_addr_t dac_offset = alpha_mv.pci_dac_offset;
204 int ok = 1;
205
206 /* If this is not set, the machine doesn't support DAC at all. */
207 if (dac_offset == 0)
208 ok = 0;
209
210 /* The device has to be able to address our DAC bit. */
211 if ((dac_offset & dev->dma_mask) != dac_offset)
212 ok = 0;
213
214 /* If both conditions above are met, we are fine. */
215 DBGA("pci_dac_dma_supported %s from %ps\n",
216 str_yes_no(ok), __builtin_return_address(0));
217
218 return ok;
219}
220
221/* Map a single buffer of the indicated size for PCI DMA in streaming
222 mode. The 32-bit PCI bus mastering address to use is returned.
223 Once the device is given the dma address, the device owns this memory
224 until either pci_unmap_single or pci_dma_sync_single is performed. */
225
226static dma_addr_t
227pci_map_single_1(struct pci_dev *pdev, phys_addr_t paddr, size_t size,
228 int dac_allowed)
229{
230 struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose;
231 dma_addr_t max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
232 unsigned long offset = offset_in_page(paddr);
233 struct pci_iommu_arena *arena;
234 long npages, dma_ofs, i;
235 dma_addr_t ret;
236 unsigned int align = 0;
237 struct device *dev = pdev ? &pdev->dev : NULL;
238
239#if !DEBUG_NODIRECT
240 /* First check to see if we can use the direct map window. */
241 if (paddr + size + __direct_map_base - 1 <= max_dma
242 && paddr + size <= __direct_map_size) {
243 ret = paddr + __direct_map_base;
244
245 DBGA2("pci_map_single: [%pa,%zx] -> direct %llx from %ps\n",
246 &paddr, size, ret, __builtin_return_address(0));
247
248 return ret;
249 }
250#endif
251
252 /* Next, use DAC if selected earlier. */
253 if (dac_allowed) {
254 ret = paddr + alpha_mv.pci_dac_offset;
255
256 DBGA2("pci_map_single: [%pa,%zx] -> DAC %llx from %ps\n",
257 &paddr, size, ret, __builtin_return_address(0));
258
259 return ret;
260 }
261
262 /* If the machine doesn't define a pci_tbi routine, we have to
263 assume it doesn't support sg mapping, and, since we tried to
264 use direct_map above, it now must be considered an error. */
265 if (! alpha_mv.mv_pci_tbi) {
266 printk_once(KERN_WARNING "pci_map_single: no HW sg\n");
267 return DMA_MAPPING_ERROR;
268 }
269
270 arena = hose->sg_pci;
271 if (!arena || arena->dma_base + arena->size - 1 > max_dma)
272 arena = hose->sg_isa;
273
274 npages = iommu_num_pages(addr: paddr, len: size, PAGE_SIZE);
275
276 /* Force allocation to 64KB boundary for ISA bridges. */
277 if (pdev && pdev == isa_bridge)
278 align = 8;
279 dma_ofs = iommu_arena_alloc(dev, arena, n: npages, align);
280 if (dma_ofs < 0) {
281 printk(KERN_WARNING "pci_map_single failed: "
282 "could not allocate dma page tables\n");
283 return DMA_MAPPING_ERROR;
284 }
285
286 paddr &= PAGE_MASK;
287 for (i = 0; i < npages; ++i, paddr += PAGE_SIZE)
288 arena->ptes[i + dma_ofs] = mk_iommu_pte(paddr);
289
290 ret = arena->dma_base + dma_ofs * PAGE_SIZE;
291 ret += offset;
292
293 DBGA2("pci_map_single: [%pa,%zx] np %ld -> sg %llx from %ps\n",
294 &paddr, size, npages, ret, __builtin_return_address(0));
295
296 return ret;
297}
298
299/* Helper for generic DMA-mapping functions. */
300static struct pci_dev *alpha_gendev_to_pci(struct device *dev)
301{
302 if (dev && dev_is_pci(dev))
303 return to_pci_dev(dev);
304
305 /* Assume that non-PCI devices asking for DMA are either ISA or EISA,
306 BUG() otherwise. */
307 BUG_ON(!isa_bridge);
308
309 /* Assume non-busmaster ISA DMA when dma_mask is not set (the ISA
310 bridge is bus master then). */
311 if (!dev || !dev->dma_mask || !*dev->dma_mask)
312 return isa_bridge;
313
314 /* For EISA bus masters, return isa_bridge (it might have smaller
315 dma_mask due to wiring limitations). */
316 if (*dev->dma_mask >= isa_bridge->dma_mask)
317 return isa_bridge;
318
319 /* This assumes ISA bus master with dma_mask 0xffffff. */
320 return NULL;
321}
322
323static dma_addr_t alpha_pci_map_phys(struct device *dev, phys_addr_t phys,
324 size_t size, enum dma_data_direction dir,
325 unsigned long attrs)
326{
327 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
328 int dac_allowed;
329
330 if (unlikely(attrs & DMA_ATTR_MMIO))
331 return DMA_MAPPING_ERROR;
332
333 dac_allowed = pdev ? pci_dac_dma_supported(dev: pdev, mask: pdev->dma_mask) : 0;
334 return pci_map_single_1(pdev, paddr: phys, size, dac_allowed);
335}
336
337/* Unmap a single streaming mode DMA translation. The DMA_ADDR and
338 SIZE must match what was provided for in a previous pci_map_single
339 call. All other usages are undefined. After this call, reads by
340 the cpu to the buffer are guaranteed to see whatever the device
341 wrote there. */
342
343static void alpha_pci_unmap_phys(struct device *dev, dma_addr_t dma_addr,
344 size_t size, enum dma_data_direction dir,
345 unsigned long attrs)
346{
347 unsigned long flags;
348 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
349 struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose;
350 struct pci_iommu_arena *arena;
351 long dma_ofs, npages;
352
353 if (dma_addr >= __direct_map_base
354 && dma_addr < __direct_map_base + __direct_map_size) {
355 /* Nothing to do. */
356
357 DBGA2("pci_unmap_single: direct [%llx,%zx] from %ps\n",
358 dma_addr, size, __builtin_return_address(0));
359
360 return;
361 }
362
363 if (dma_addr > 0xffffffff) {
364 DBGA2("pci64_unmap_single: DAC [%llx,%zx] from %ps\n",
365 dma_addr, size, __builtin_return_address(0));
366 return;
367 }
368
369 arena = hose->sg_pci;
370 if (!arena || dma_addr < arena->dma_base)
371 arena = hose->sg_isa;
372
373 dma_ofs = (dma_addr - arena->dma_base) >> PAGE_SHIFT;
374 if (dma_ofs * PAGE_SIZE >= arena->size) {
375 printk(KERN_ERR "Bogus pci_unmap_single: dma_addr %llx "
376 " base %llx size %x\n",
377 dma_addr, arena->dma_base, arena->size);
378 return;
379 BUG();
380 }
381
382 npages = iommu_num_pages(addr: dma_addr, len: size, PAGE_SIZE);
383
384 spin_lock_irqsave(&arena->lock, flags);
385
386 iommu_arena_free(arena, ofs: dma_ofs, n: npages);
387
388 /* If we're freeing ptes above the `next_entry' pointer (they
389 may have snuck back into the TLB since the last wrap flush),
390 we need to flush the TLB before reallocating the latter. */
391 if (dma_ofs >= arena->next_entry)
392 alpha_mv.mv_pci_tbi(hose, dma_addr, dma_addr + size - 1);
393
394 spin_unlock_irqrestore(lock: &arena->lock, flags);
395
396 DBGA2("pci_unmap_single: sg [%llx,%zx] np %ld from %ps\n",
397 dma_addr, size, npages, __builtin_return_address(0));
398}
399
400/* Allocate and map kernel buffer using consistent mode DMA for PCI
401 device. Returns non-NULL cpu-view pointer to the buffer if
402 successful and sets *DMA_ADDRP to the pci side dma address as well,
403 else DMA_ADDRP is undefined. */
404
405static void *alpha_pci_alloc_coherent(struct device *dev, size_t size,
406 dma_addr_t *dma_addrp, gfp_t gfp,
407 unsigned long attrs)
408{
409 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
410 void *cpu_addr;
411 long order = get_order(size);
412
413 gfp &= ~GFP_DMA;
414
415try_again:
416 cpu_addr = (void *)__get_free_pages(gfp | __GFP_ZERO, order);
417 if (! cpu_addr) {
418 printk(KERN_INFO "pci_alloc_consistent: "
419 "get_free_pages failed from %ps\n",
420 __builtin_return_address(0));
421 /* ??? Really atomic allocation? Otherwise we could play
422 with vmalloc and sg if we can't find contiguous memory. */
423 return NULL;
424 }
425 memset(cpu_addr, 0, size);
426
427 *dma_addrp = pci_map_single_1(pdev, virt_to_phys(address: cpu_addr), size, dac_allowed: 0);
428 if (*dma_addrp == DMA_MAPPING_ERROR) {
429 free_pages(addr: (unsigned long)cpu_addr, order);
430 if (alpha_mv.mv_pci_tbi || (gfp & GFP_DMA))
431 return NULL;
432 /* The address doesn't fit required mask and we
433 do not have iommu. Try again with GFP_DMA. */
434 gfp |= GFP_DMA;
435 goto try_again;
436 }
437
438 DBGA2("pci_alloc_consistent: %zx -> [%p,%llx] from %ps\n",
439 size, cpu_addr, *dma_addrp, __builtin_return_address(0));
440
441 return cpu_addr;
442}
443
444/* Free and unmap a consistent DMA buffer. CPU_ADDR and DMA_ADDR must
445 be values that were returned from pci_alloc_consistent. SIZE must
446 be the same as what as passed into pci_alloc_consistent.
447 References to the memory and mappings associated with CPU_ADDR or
448 DMA_ADDR past this call are illegal. */
449
450static void alpha_pci_free_coherent(struct device *dev, size_t size,
451 void *cpu_addr, dma_addr_t dma_addr,
452 unsigned long attrs)
453{
454 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
455 dma_unmap_single(&pdev->dev, dma_addr, size, DMA_BIDIRECTIONAL);
456 free_pages(addr: (unsigned long)cpu_addr, order: get_order(size));
457
458 DBGA2("pci_free_consistent: [%llx,%zx] from %ps\n",
459 dma_addr, size, __builtin_return_address(0));
460}
461
462/* Classify the elements of the scatterlist. Write dma_address
463 of each element with:
464 0 : Followers all physically adjacent.
465 1 : Followers all virtually adjacent.
466 -1 : Not leader, physically adjacent to previous.
467 -2 : Not leader, virtually adjacent to previous.
468 Write dma_length of each leader with the combined lengths of
469 the mergable followers. */
470
471#define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG)))
472#define SG_ENT_PHYS_ADDRESS(SG) __pa(SG_ENT_VIRT_ADDRESS(SG))
473
474static void
475sg_classify(struct device *dev, struct scatterlist *sg, struct scatterlist *end,
476 int virt_ok)
477{
478 unsigned long next_paddr;
479 struct scatterlist *leader;
480 long leader_flag, leader_length;
481 unsigned int max_seg_size;
482
483 leader = sg;
484 leader_flag = 0;
485 leader_length = leader->length;
486 next_paddr = SG_ENT_PHYS_ADDRESS(leader) + leader_length;
487
488 /* we will not marge sg without device. */
489 max_seg_size = dev ? dma_get_max_seg_size(dev) : 0;
490 for (++sg; sg < end; ++sg) {
491 unsigned long addr, len;
492 addr = SG_ENT_PHYS_ADDRESS(sg);
493 len = sg->length;
494
495 if (leader_length + len > max_seg_size)
496 goto new_segment;
497
498 if (next_paddr == addr) {
499 sg->dma_address = -1;
500 leader_length += len;
501 } else if (((next_paddr | addr) & ~PAGE_MASK) == 0 && virt_ok) {
502 sg->dma_address = -2;
503 leader_flag = 1;
504 leader_length += len;
505 } else {
506new_segment:
507 leader->dma_address = leader_flag;
508 leader->dma_length = leader_length;
509 leader = sg;
510 leader_flag = 0;
511 leader_length = len;
512 }
513
514 next_paddr = addr + len;
515 }
516
517 leader->dma_address = leader_flag;
518 leader->dma_length = leader_length;
519}
520
521/* Given a scatterlist leader, choose an allocation method and fill
522 in the blanks. */
523
524static int
525sg_fill(struct device *dev, struct scatterlist *leader, struct scatterlist *end,
526 struct scatterlist *out, struct pci_iommu_arena *arena,
527 dma_addr_t max_dma, int dac_allowed)
528{
529 unsigned long paddr = SG_ENT_PHYS_ADDRESS(leader);
530 long size = leader->dma_length;
531 struct scatterlist *sg;
532 unsigned long *ptes;
533 long npages, dma_ofs, i;
534
535#if !DEBUG_NODIRECT
536 /* If everything is physically contiguous, and the addresses
537 fall into the direct-map window, use it. */
538 if (leader->dma_address == 0
539 && paddr + size + __direct_map_base - 1 <= max_dma
540 && paddr + size <= __direct_map_size) {
541 out->dma_address = paddr + __direct_map_base;
542 out->dma_length = size;
543
544 DBGA(" sg_fill: [%p,%lx] -> direct %llx\n",
545 __va(paddr), size, out->dma_address);
546
547 return 0;
548 }
549#endif
550
551 /* If physically contiguous and DAC is available, use it. */
552 if (leader->dma_address == 0 && dac_allowed) {
553 out->dma_address = paddr + alpha_mv.pci_dac_offset;
554 out->dma_length = size;
555
556 DBGA(" sg_fill: [%p,%lx] -> DAC %llx\n",
557 __va(paddr), size, out->dma_address);
558
559 return 0;
560 }
561
562 /* Otherwise, we'll use the iommu to make the pages virtually
563 contiguous. */
564
565 paddr &= ~PAGE_MASK;
566 npages = iommu_num_pages(addr: paddr, len: size, PAGE_SIZE);
567 dma_ofs = iommu_arena_alloc(dev, arena, n: npages, align: 0);
568 if (dma_ofs < 0) {
569 /* If we attempted a direct map above but failed, die. */
570 if (leader->dma_address == 0)
571 return -1;
572
573 /* Otherwise, break up the remaining virtually contiguous
574 hunks into individual direct maps and retry. */
575 sg_classify(dev, sg: leader, end, virt_ok: 0);
576 return sg_fill(dev, leader, end, out, arena, max_dma, dac_allowed);
577 }
578
579 out->dma_address = arena->dma_base + dma_ofs*PAGE_SIZE + paddr;
580 out->dma_length = size;
581
582 DBGA(" sg_fill: [%p,%lx] -> sg %llx np %ld\n",
583 __va(paddr), size, out->dma_address, npages);
584
585 /* All virtually contiguous. We need to find the length of each
586 physically contiguous subsegment to fill in the ptes. */
587 ptes = &arena->ptes[dma_ofs];
588 sg = leader;
589 do {
590#if DEBUG_ALLOC > 0
591 struct scatterlist *last_sg = sg;
592#endif
593
594 size = sg->length;
595 paddr = SG_ENT_PHYS_ADDRESS(sg);
596
597 while (sg+1 < end && (int) sg[1].dma_address == -1) {
598 size += sg[1].length;
599 sg = sg_next(sg);
600 }
601
602 npages = iommu_num_pages(addr: paddr, len: size, PAGE_SIZE);
603
604 paddr &= PAGE_MASK;
605 for (i = 0; i < npages; ++i, paddr += PAGE_SIZE)
606 *ptes++ = mk_iommu_pte(paddr);
607
608#if DEBUG_ALLOC > 0
609 DBGA(" (%ld) [%p,%x] np %ld\n",
610 last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
611 last_sg->length, npages);
612 while (++last_sg <= sg) {
613 DBGA(" (%ld) [%p,%x] cont\n",
614 last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
615 last_sg->length);
616 }
617#endif
618 } while (++sg < end && (int) sg->dma_address < 0);
619
620 return 1;
621}
622
623static int alpha_pci_map_sg(struct device *dev, struct scatterlist *sg,
624 int nents, enum dma_data_direction dir,
625 unsigned long attrs)
626{
627 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
628 struct scatterlist *start, *end, *out;
629 struct pci_controller *hose;
630 struct pci_iommu_arena *arena;
631 dma_addr_t max_dma;
632 int dac_allowed;
633
634 BUG_ON(dir == DMA_NONE);
635
636 dac_allowed = dev ? pci_dac_dma_supported(dev: pdev, mask: pdev->dma_mask) : 0;
637
638 /* Fast path single entry scatterlists. */
639 if (nents == 1) {
640 sg->dma_length = sg->length;
641 sg->dma_address = pci_map_single_1(pdev, paddr: sg_phys(sg),
642 size: sg->length, dac_allowed);
643 if (sg->dma_address == DMA_MAPPING_ERROR)
644 return -EIO;
645 return 1;
646 }
647
648 start = sg;
649 end = sg + nents;
650
651 /* First, prepare information about the entries. */
652 sg_classify(dev, sg, end, virt_ok: alpha_mv.mv_pci_tbi != 0);
653
654 /* Second, figure out where we're going to map things. */
655 if (alpha_mv.mv_pci_tbi) {
656 hose = pdev ? pdev->sysdata : pci_isa_hose;
657 max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
658 arena = hose->sg_pci;
659 if (!arena || arena->dma_base + arena->size - 1 > max_dma)
660 arena = hose->sg_isa;
661 } else {
662 max_dma = -1;
663 arena = NULL;
664 hose = NULL;
665 }
666
667 /* Third, iterate over the scatterlist leaders and allocate
668 dma space as needed. */
669 for (out = sg; sg < end; ++sg) {
670 if ((int) sg->dma_address < 0)
671 continue;
672 if (sg_fill(dev, leader: sg, end, out, arena, max_dma, dac_allowed) < 0)
673 goto error;
674 out++;
675 }
676
677 /* Mark the end of the list for pci_unmap_sg. */
678 if (out < end)
679 out->dma_length = 0;
680
681 if (out - start == 0) {
682 printk(KERN_WARNING "pci_map_sg failed: no entries?\n");
683 return -ENOMEM;
684 }
685 DBGA("pci_map_sg: %ld entries\n", out - start);
686
687 return out - start;
688
689 error:
690 printk(KERN_WARNING "pci_map_sg failed: "
691 "could not allocate dma page tables\n");
692
693 /* Some allocation failed while mapping the scatterlist
694 entries. Unmap them now. */
695 if (out > start)
696 dma_unmap_sg(&pdev->dev, start, out - start, dir);
697 return -ENOMEM;
698}
699
700/* Unmap a set of streaming mode DMA translations. Again, cpu read
701 rules concerning calls here are the same as for pci_unmap_single()
702 above. */
703
704static void alpha_pci_unmap_sg(struct device *dev, struct scatterlist *sg,
705 int nents, enum dma_data_direction dir,
706 unsigned long attrs)
707{
708 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
709 unsigned long flags;
710 struct pci_controller *hose;
711 struct pci_iommu_arena *arena;
712 struct scatterlist *end;
713 dma_addr_t max_dma;
714 dma_addr_t fbeg, fend;
715
716 BUG_ON(dir == DMA_NONE);
717
718 if (! alpha_mv.mv_pci_tbi)
719 return;
720
721 hose = pdev ? pdev->sysdata : pci_isa_hose;
722 max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
723 arena = hose->sg_pci;
724 if (!arena || arena->dma_base + arena->size - 1 > max_dma)
725 arena = hose->sg_isa;
726
727 fbeg = -1, fend = 0;
728
729 spin_lock_irqsave(&arena->lock, flags);
730
731 for (end = sg + nents; sg < end; ++sg) {
732 dma_addr_t addr;
733 size_t size;
734 long npages, ofs;
735 dma_addr_t tend;
736
737 addr = sg->dma_address;
738 size = sg->dma_length;
739 if (!size)
740 break;
741
742 if (addr > 0xffffffff) {
743 /* It's a DAC address -- nothing to do. */
744 DBGA(" (%ld) DAC [%llx,%zx]\n",
745 sg - end + nents, addr, size);
746 continue;
747 }
748
749 if (addr >= __direct_map_base
750 && addr < __direct_map_base + __direct_map_size) {
751 /* Nothing to do. */
752 DBGA(" (%ld) direct [%llx,%zx]\n",
753 sg - end + nents, addr, size);
754 continue;
755 }
756
757 DBGA(" (%ld) sg [%llx,%zx]\n",
758 sg - end + nents, addr, size);
759
760 npages = iommu_num_pages(addr, len: size, PAGE_SIZE);
761 ofs = (addr - arena->dma_base) >> PAGE_SHIFT;
762 iommu_arena_free(arena, ofs, n: npages);
763
764 tend = addr + size - 1;
765 if (fbeg > addr) fbeg = addr;
766 if (fend < tend) fend = tend;
767 }
768
769 /* If we're freeing ptes above the `next_entry' pointer (they
770 may have snuck back into the TLB since the last wrap flush),
771 we need to flush the TLB before reallocating the latter. */
772 if ((fend - arena->dma_base) >> PAGE_SHIFT >= arena->next_entry)
773 alpha_mv.mv_pci_tbi(hose, fbeg, fend);
774
775 spin_unlock_irqrestore(lock: &arena->lock, flags);
776
777 DBGA("pci_unmap_sg: %ld entries\n", nents - (end - sg));
778}
779
780/* Return whether the given PCI device DMA address mask can be
781 supported properly. */
782
783static int alpha_pci_supported(struct device *dev, u64 mask)
784{
785 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
786 struct pci_controller *hose;
787 struct pci_iommu_arena *arena;
788
789 /* If there exists a direct map, and the mask fits either
790 the entire direct mapped space or the total system memory as
791 shifted by the map base */
792 if (__direct_map_size != 0
793 && (__direct_map_base + __direct_map_size - 1 <= mask ||
794 __direct_map_base + (max_low_pfn << PAGE_SHIFT) - 1 <= mask))
795 return 1;
796
797 /* Check that we have a scatter-gather arena that fits. */
798 hose = pdev ? pdev->sysdata : pci_isa_hose;
799 arena = hose->sg_isa;
800 if (arena && arena->dma_base + arena->size - 1 <= mask)
801 return 1;
802 arena = hose->sg_pci;
803 if (arena && arena->dma_base + arena->size - 1 <= mask)
804 return 1;
805
806 /* As last resort try ZONE_DMA. */
807 if (!__direct_map_base && MAX_DMA_ADDRESS - IDENT_ADDR - 1 <= mask)
808 return 1;
809
810 return 0;
811}
812
813
814/*
815 * AGP GART extensions to the IOMMU
816 */
817int
818iommu_reserve(struct pci_iommu_arena *arena, long pg_count, long align_mask)
819{
820 unsigned long flags;
821 unsigned long *ptes;
822 long i, p;
823
824 if (!arena) return -EINVAL;
825
826 spin_lock_irqsave(&arena->lock, flags);
827
828 /* Search for N empty ptes. */
829 ptes = arena->ptes;
830 p = iommu_arena_find_pages(NULL, arena, n: pg_count, mask: align_mask);
831 if (p < 0) {
832 spin_unlock_irqrestore(lock: &arena->lock, flags);
833 return -1;
834 }
835
836 /* Success. Mark them all reserved (ie not zero and invalid)
837 for the iommu tlb that could load them from under us.
838 They will be filled in with valid bits by _bind() */
839 for (i = 0; i < pg_count; ++i)
840 ptes[p+i] = IOMMU_RESERVED_PTE;
841
842 arena->next_entry = p + pg_count;
843 spin_unlock_irqrestore(lock: &arena->lock, flags);
844
845 return p;
846}
847
848int
849iommu_release(struct pci_iommu_arena *arena, long pg_start, long pg_count)
850{
851 unsigned long *ptes;
852 long i;
853
854 if (!arena) return -EINVAL;
855
856 ptes = arena->ptes;
857
858 /* Make sure they're all reserved first... */
859 for(i = pg_start; i < pg_start + pg_count; i++)
860 if (ptes[i] != IOMMU_RESERVED_PTE)
861 return -EBUSY;
862
863 iommu_arena_free(arena, ofs: pg_start, n: pg_count);
864 return 0;
865}
866
867int
868iommu_bind(struct pci_iommu_arena *arena, long pg_start, long pg_count,
869 struct page **pages)
870{
871 unsigned long flags;
872 unsigned long *ptes;
873 long i, j;
874
875 if (!arena) return -EINVAL;
876
877 spin_lock_irqsave(&arena->lock, flags);
878
879 ptes = arena->ptes;
880
881 for(j = pg_start; j < pg_start + pg_count; j++) {
882 if (ptes[j] != IOMMU_RESERVED_PTE) {
883 spin_unlock_irqrestore(lock: &arena->lock, flags);
884 return -EBUSY;
885 }
886 }
887
888 for(i = 0, j = pg_start; i < pg_count; i++, j++)
889 ptes[j] = mk_iommu_pte(page_to_phys(pages[i]));
890
891 spin_unlock_irqrestore(lock: &arena->lock, flags);
892
893 return 0;
894}
895
896int
897iommu_unbind(struct pci_iommu_arena *arena, long pg_start, long pg_count)
898{
899 unsigned long *p;
900 long i;
901
902 if (!arena) return -EINVAL;
903
904 p = arena->ptes + pg_start;
905 for(i = 0; i < pg_count; i++)
906 p[i] = IOMMU_RESERVED_PTE;
907
908 return 0;
909}
910
911const struct dma_map_ops alpha_pci_ops = {
912 .alloc = alpha_pci_alloc_coherent,
913 .free = alpha_pci_free_coherent,
914 .map_phys = alpha_pci_map_phys,
915 .unmap_phys = alpha_pci_unmap_phys,
916 .map_sg = alpha_pci_map_sg,
917 .unmap_sg = alpha_pci_unmap_sg,
918 .dma_supported = alpha_pci_supported,
919 .mmap = dma_common_mmap,
920 .get_sgtable = dma_common_get_sgtable,
921 .alloc_pages_op = dma_common_alloc_pages,
922 .free_pages = dma_common_free_pages,
923};
924EXPORT_SYMBOL(alpha_pci_ops);
925

source code of linux/arch/alpha/kernel/pci_iommu.c