1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Microsemi Switchtec(tm) PCIe Management Driver
4 * Copyright (c) 2017, Microsemi Corporation
5 */
6
7#include <linux/interrupt.h>
8#include <linux/io-64-nonatomic-lo-hi.h>
9#include <linux/delay.h>
10#include <linux/kthread.h>
11#include <linux/module.h>
12#include <linux/ntb.h>
13#include <linux/pci.h>
14#include <linux/switchtec.h>
15
16MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver");
17MODULE_VERSION("0.1");
18MODULE_LICENSE("GPL");
19MODULE_AUTHOR("Microsemi Corporation");
20
21static ulong max_mw_size = SZ_2M;
22module_param(max_mw_size, ulong, 0644);
23MODULE_PARM_DESC(max_mw_size,
24 "Max memory window size reported to the upper layer");
25
26static bool use_lut_mws;
27module_param(use_lut_mws, bool, 0644);
28MODULE_PARM_DESC(use_lut_mws,
29 "Enable the use of the LUT based memory windows");
30
31#define SWITCHTEC_NTB_MAGIC 0x45CC0001
32#define MAX_MWS 128
33
34struct shared_mw {
35 u32 magic;
36 u32 link_sta;
37 u32 partition_id;
38 u64 mw_sizes[MAX_MWS];
39 u32 spad[128];
40};
41
42#define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry)
43#define LUT_SIZE SZ_64K
44
45struct switchtec_ntb {
46 struct ntb_dev ntb;
47 struct switchtec_dev *stdev;
48
49 int self_partition;
50 int peer_partition;
51
52 int doorbell_irq;
53 int message_irq;
54
55 struct ntb_info_regs __iomem *mmio_ntb;
56 struct ntb_ctrl_regs __iomem *mmio_ctrl;
57 struct ntb_dbmsg_regs __iomem *mmio_dbmsg;
58 struct ntb_ctrl_regs __iomem *mmio_self_ctrl;
59 struct ntb_ctrl_regs __iomem *mmio_peer_ctrl;
60 struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg;
61 struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg;
62
63 void __iomem *mmio_xlink_win;
64
65 struct shared_mw *self_shared;
66 struct shared_mw __iomem *peer_shared;
67 dma_addr_t self_shared_dma;
68
69 u64 db_mask;
70 u64 db_valid_mask;
71 int db_shift;
72 int db_peer_shift;
73
74 /* synchronize rmw access of db_mask and hw reg */
75 spinlock_t db_mask_lock;
76
77 int nr_direct_mw;
78 int nr_lut_mw;
79 int nr_rsvd_luts;
80 int direct_mw_to_bar[MAX_DIRECT_MW];
81
82 int peer_nr_direct_mw;
83 int peer_nr_lut_mw;
84 int peer_direct_mw_to_bar[MAX_DIRECT_MW];
85
86 bool link_is_up;
87 enum ntb_speed link_speed;
88 enum ntb_width link_width;
89 struct work_struct check_link_status_work;
90 bool link_force_down;
91};
92
93static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb)
94{
95 return container_of(ntb, struct switchtec_ntb, ntb);
96}
97
98static int switchtec_ntb_part_op(struct switchtec_ntb *sndev,
99 struct ntb_ctrl_regs __iomem *ctl,
100 u32 op, int wait_status)
101{
102 static const char * const op_text[] = {
103 [NTB_CTRL_PART_OP_LOCK] = "lock",
104 [NTB_CTRL_PART_OP_CFG] = "configure",
105 [NTB_CTRL_PART_OP_RESET] = "reset",
106 };
107
108 int i;
109 u32 ps;
110 int status;
111
112 switch (op) {
113 case NTB_CTRL_PART_OP_LOCK:
114 status = NTB_CTRL_PART_STATUS_LOCKING;
115 break;
116 case NTB_CTRL_PART_OP_CFG:
117 status = NTB_CTRL_PART_STATUS_CONFIGURING;
118 break;
119 case NTB_CTRL_PART_OP_RESET:
120 status = NTB_CTRL_PART_STATUS_RESETTING;
121 break;
122 default:
123 return -EINVAL;
124 }
125
126 iowrite32(op, &ctl->partition_op);
127
128 for (i = 0; i < 1000; i++) {
129 if (msleep_interruptible(msecs: 50) != 0) {
130 iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op);
131 return -EINTR;
132 }
133
134 ps = ioread32(&ctl->partition_status) & 0xFFFF;
135
136 if (ps != status)
137 break;
138 }
139
140 if (ps == wait_status)
141 return 0;
142
143 if (ps == status) {
144 dev_err(&sndev->stdev->dev,
145 "Timed out while performing %s (%d). (%08x)\n",
146 op_text[op], op,
147 ioread32(&ctl->partition_status));
148
149 return -ETIMEDOUT;
150 }
151
152 return -EIO;
153}
154
155static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx,
156 u32 val)
157{
158 if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg))
159 return -EINVAL;
160
161 iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg);
162
163 return 0;
164}
165
166static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx)
167{
168 struct switchtec_ntb *sndev = ntb_sndev(ntb);
169 int nr_direct_mw = sndev->peer_nr_direct_mw;
170 int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts;
171
172 if (pidx != NTB_DEF_PEER_IDX)
173 return -EINVAL;
174
175 if (!use_lut_mws)
176 nr_lut_mw = 0;
177
178 return nr_direct_mw + nr_lut_mw;
179}
180
181static int lut_index(struct switchtec_ntb *sndev, int mw_idx)
182{
183 return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts;
184}
185
186static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx)
187{
188 return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts;
189}
190
191static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx,
192 int widx, resource_size_t *addr_align,
193 resource_size_t *size_align,
194 resource_size_t *size_max)
195{
196 struct switchtec_ntb *sndev = ntb_sndev(ntb);
197 int lut;
198 resource_size_t size;
199
200 if (pidx != NTB_DEF_PEER_IDX)
201 return -EINVAL;
202
203 lut = widx >= sndev->peer_nr_direct_mw;
204 size = ioread64(addr: &sndev->peer_shared->mw_sizes[widx]);
205
206 if (size == 0)
207 return -EINVAL;
208
209 if (addr_align)
210 *addr_align = lut ? size : SZ_4K;
211
212 if (size_align)
213 *size_align = lut ? size : SZ_4K;
214
215 if (size_max)
216 *size_max = size;
217
218 return 0;
219}
220
221static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx)
222{
223 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
224 int bar = sndev->peer_direct_mw_to_bar[idx];
225 u32 ctl_val;
226
227 ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
228 ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN;
229 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
230 iowrite32(0, &ctl->bar_entry[bar].win_size);
231 iowrite32(0, &ctl->bar_ext_entry[bar].win_size);
232 iowrite64(val: sndev->self_partition, addr: &ctl->bar_entry[bar].xlate_addr);
233}
234
235static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx)
236{
237 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
238
239 iowrite64(val: 0, addr: &ctl->lut_entry[peer_lut_index(sndev, mw_idx: idx)]);
240}
241
242static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx,
243 dma_addr_t addr, resource_size_t size)
244{
245 int xlate_pos = ilog2(size);
246 int bar = sndev->peer_direct_mw_to_bar[idx];
247 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
248 u32 ctl_val;
249
250 ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
251 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
252
253 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
254 iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
255 &ctl->bar_entry[bar].win_size);
256 iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
257 iowrite64(val: sndev->self_partition | addr,
258 addr: &ctl->bar_entry[bar].xlate_addr);
259}
260
261static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx,
262 dma_addr_t addr, resource_size_t size)
263{
264 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
265
266 iowrite64(val: (NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr),
267 addr: &ctl->lut_entry[peer_lut_index(sndev, mw_idx: idx)]);
268}
269
270static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
271 dma_addr_t addr, resource_size_t size)
272{
273 struct switchtec_ntb *sndev = ntb_sndev(ntb);
274 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
275 int xlate_pos = ilog2(size);
276 int nr_direct_mw = sndev->peer_nr_direct_mw;
277 int rc;
278
279 if (pidx != NTB_DEF_PEER_IDX)
280 return -EINVAL;
281
282 dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n",
283 widx, pidx, &addr, &size);
284
285 if (widx >= switchtec_ntb_mw_count(ntb, pidx))
286 return -EINVAL;
287
288 if (size != 0 && xlate_pos < 12)
289 return -EINVAL;
290
291 if (xlate_pos >= 0 && !IS_ALIGNED(addr, BIT_ULL(xlate_pos))) {
292 /*
293 * In certain circumstances we can get a buffer that is
294 * not aligned to its size. (Most of the time
295 * dma_alloc_coherent ensures this). This can happen when
296 * using large buffers allocated by the CMA
297 * (see CMA_CONFIG_ALIGNMENT)
298 */
299 dev_err(&sndev->stdev->dev,
300 "ERROR: Memory window address is not aligned to its size!\n");
301 return -EINVAL;
302 }
303
304 rc = switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_LOCK,
305 wait_status: NTB_CTRL_PART_STATUS_LOCKED);
306 if (rc)
307 return rc;
308
309 if (size == 0) {
310 if (widx < nr_direct_mw)
311 switchtec_ntb_mw_clr_direct(sndev, idx: widx);
312 else
313 switchtec_ntb_mw_clr_lut(sndev, idx: widx);
314 } else {
315 if (widx < nr_direct_mw)
316 switchtec_ntb_mw_set_direct(sndev, idx: widx, addr, size);
317 else
318 switchtec_ntb_mw_set_lut(sndev, idx: widx, addr, size);
319 }
320
321 rc = switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_CFG,
322 wait_status: NTB_CTRL_PART_STATUS_NORMAL);
323
324 if (rc == -EIO) {
325 dev_err(&sndev->stdev->dev,
326 "Hardware reported an error configuring mw %d: %08x\n",
327 widx, ioread32(&ctl->bar_error));
328
329 if (widx < nr_direct_mw)
330 switchtec_ntb_mw_clr_direct(sndev, idx: widx);
331 else
332 switchtec_ntb_mw_clr_lut(sndev, idx: widx);
333
334 switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_CFG,
335 wait_status: NTB_CTRL_PART_STATUS_NORMAL);
336 }
337
338 return rc;
339}
340
341static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb)
342{
343 struct switchtec_ntb *sndev = ntb_sndev(ntb);
344 int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts;
345
346 return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0);
347}
348
349static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev,
350 int idx, phys_addr_t *base,
351 resource_size_t *size)
352{
353 int bar = sndev->direct_mw_to_bar[idx];
354 size_t offset = 0;
355
356 if (bar < 0)
357 return -EINVAL;
358
359 if (idx == 0) {
360 /*
361 * This is the direct BAR shared with the LUTs
362 * which means the actual window will be offset
363 * by the size of all the LUT entries.
364 */
365
366 offset = LUT_SIZE * sndev->nr_lut_mw;
367 }
368
369 if (base)
370 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
371
372 if (size) {
373 *size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
374 if (offset && *size > offset)
375 *size = offset;
376
377 if (*size > max_mw_size)
378 *size = max_mw_size;
379 }
380
381 return 0;
382}
383
384static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev,
385 int idx, phys_addr_t *base,
386 resource_size_t *size)
387{
388 int bar = sndev->direct_mw_to_bar[0];
389 int offset;
390
391 offset = LUT_SIZE * lut_index(sndev, mw_idx: idx);
392
393 if (base)
394 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
395
396 if (size)
397 *size = LUT_SIZE;
398
399 return 0;
400}
401
402static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
403 phys_addr_t *base,
404 resource_size_t *size)
405{
406 struct switchtec_ntb *sndev = ntb_sndev(ntb);
407
408 if (idx < sndev->nr_direct_mw)
409 return switchtec_ntb_direct_get_addr(sndev, idx, base, size);
410 else if (idx < switchtec_ntb_peer_mw_count(ntb))
411 return switchtec_ntb_lut_get_addr(sndev, idx, base, size);
412 else
413 return -EINVAL;
414}
415
416static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev,
417 int partition,
418 enum ntb_speed *speed,
419 enum ntb_width *width)
420{
421 struct switchtec_dev *stdev = sndev->stdev;
422 struct part_cfg_regs __iomem *part_cfg =
423 &stdev->mmio_part_cfg_all[partition];
424
425 u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF;
426 u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]);
427
428 if (speed)
429 *speed = (linksta >> 16) & 0xF;
430
431 if (width)
432 *width = (linksta >> 20) & 0x3F;
433}
434
435static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev)
436{
437 enum ntb_speed self_speed, peer_speed;
438 enum ntb_width self_width, peer_width;
439
440 if (!sndev->link_is_up) {
441 sndev->link_speed = NTB_SPEED_NONE;
442 sndev->link_width = NTB_WIDTH_NONE;
443 return;
444 }
445
446 switchtec_ntb_part_link_speed(sndev, partition: sndev->self_partition,
447 speed: &self_speed, width: &self_width);
448 switchtec_ntb_part_link_speed(sndev, partition: sndev->peer_partition,
449 speed: &peer_speed, width: &peer_width);
450
451 sndev->link_speed = min(self_speed, peer_speed);
452 sndev->link_width = min(self_width, peer_width);
453}
454
455static int crosslink_is_enabled(struct switchtec_ntb *sndev)
456{
457 struct ntb_info_regs __iomem *inf = sndev->mmio_ntb;
458
459 return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled);
460}
461
462static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev)
463{
464 int i;
465 u32 msg_map = 0;
466
467 if (!crosslink_is_enabled(sndev))
468 return;
469
470 for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) {
471 int m = i | sndev->self_partition << 2;
472
473 msg_map |= m << i * 8;
474 }
475
476 iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map);
477 iowrite64(val: sndev->db_valid_mask << sndev->db_peer_shift,
478 addr: &sndev->mmio_peer_dbmsg->odb_mask);
479}
480
481enum switchtec_msg {
482 LINK_MESSAGE = 0,
483 MSG_LINK_UP = 1,
484 MSG_LINK_DOWN = 2,
485 MSG_CHECK_LINK = 3,
486 MSG_LINK_FORCE_DOWN = 4,
487};
488
489static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev);
490
491static void switchtec_ntb_link_status_update(struct switchtec_ntb *sndev)
492{
493 int link_sta;
494 int old = sndev->link_is_up;
495
496 link_sta = sndev->self_shared->link_sta;
497 if (link_sta) {
498 u64 peer = ioread64(addr: &sndev->peer_shared->magic);
499
500 if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC)
501 link_sta = peer >> 32;
502 else
503 link_sta = 0;
504 }
505
506 sndev->link_is_up = link_sta;
507 switchtec_ntb_set_link_speed(sndev);
508
509 if (link_sta != old) {
510 switchtec_ntb_send_msg(sndev, idx: LINK_MESSAGE, val: MSG_CHECK_LINK);
511 ntb_link_event(ntb: &sndev->ntb);
512 dev_info(&sndev->stdev->dev, "ntb link %s\n",
513 link_sta ? "up" : "down");
514
515 if (link_sta)
516 crosslink_init_dbmsgs(sndev);
517 }
518}
519
520static void check_link_status_work(struct work_struct *work)
521{
522 struct switchtec_ntb *sndev;
523
524 sndev = container_of(work, struct switchtec_ntb,
525 check_link_status_work);
526
527 if (sndev->link_force_down) {
528 sndev->link_force_down = false;
529 switchtec_ntb_reinit_peer(sndev);
530
531 if (sndev->link_is_up) {
532 sndev->link_is_up = 0;
533 ntb_link_event(ntb: &sndev->ntb);
534 dev_info(&sndev->stdev->dev, "ntb link forced down\n");
535 }
536
537 return;
538 }
539
540 switchtec_ntb_link_status_update(sndev);
541}
542
543static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,
544 enum switchtec_msg msg)
545{
546 if (msg == MSG_LINK_FORCE_DOWN)
547 sndev->link_force_down = true;
548
549 schedule_work(work: &sndev->check_link_status_work);
550}
551
552static void switchtec_ntb_link_notification(struct switchtec_dev *stdev)
553{
554 struct switchtec_ntb *sndev = stdev->sndev;
555
556 switchtec_ntb_check_link(sndev, msg: MSG_CHECK_LINK);
557}
558
559static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb,
560 enum ntb_speed *speed,
561 enum ntb_width *width)
562{
563 struct switchtec_ntb *sndev = ntb_sndev(ntb);
564
565 if (speed)
566 *speed = sndev->link_speed;
567 if (width)
568 *width = sndev->link_width;
569
570 return sndev->link_is_up;
571}
572
573static int switchtec_ntb_link_enable(struct ntb_dev *ntb,
574 enum ntb_speed max_speed,
575 enum ntb_width max_width)
576{
577 struct switchtec_ntb *sndev = ntb_sndev(ntb);
578
579 dev_dbg(&sndev->stdev->dev, "enabling link\n");
580
581 sndev->self_shared->link_sta = 1;
582 switchtec_ntb_send_msg(sndev, idx: LINK_MESSAGE, val: MSG_LINK_UP);
583
584 switchtec_ntb_link_status_update(sndev);
585
586 return 0;
587}
588
589static int switchtec_ntb_link_disable(struct ntb_dev *ntb)
590{
591 struct switchtec_ntb *sndev = ntb_sndev(ntb);
592
593 dev_dbg(&sndev->stdev->dev, "disabling link\n");
594
595 sndev->self_shared->link_sta = 0;
596 switchtec_ntb_send_msg(sndev, idx: LINK_MESSAGE, val: MSG_LINK_DOWN);
597
598 switchtec_ntb_link_status_update(sndev);
599
600 return 0;
601}
602
603static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb)
604{
605 struct switchtec_ntb *sndev = ntb_sndev(ntb);
606
607 return sndev->db_valid_mask;
608}
609
610static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb)
611{
612 return 1;
613}
614
615static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
616{
617 struct switchtec_ntb *sndev = ntb_sndev(ntb);
618
619 if (db_vector < 0 || db_vector > 1)
620 return 0;
621
622 return sndev->db_valid_mask;
623}
624
625static u64 switchtec_ntb_db_read(struct ntb_dev *ntb)
626{
627 u64 ret;
628 struct switchtec_ntb *sndev = ntb_sndev(ntb);
629
630 ret = ioread64(addr: &sndev->mmio_self_dbmsg->idb) >> sndev->db_shift;
631
632 return ret & sndev->db_valid_mask;
633}
634
635static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
636{
637 struct switchtec_ntb *sndev = ntb_sndev(ntb);
638
639 iowrite64(val: db_bits << sndev->db_shift, addr: &sndev->mmio_self_dbmsg->idb);
640
641 return 0;
642}
643
644static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
645{
646 unsigned long irqflags;
647 struct switchtec_ntb *sndev = ntb_sndev(ntb);
648
649 if (db_bits & ~sndev->db_valid_mask)
650 return -EINVAL;
651
652 spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
653
654 sndev->db_mask |= db_bits << sndev->db_shift;
655 iowrite64(val: ~sndev->db_mask, addr: &sndev->mmio_self_dbmsg->idb_mask);
656
657 spin_unlock_irqrestore(lock: &sndev->db_mask_lock, flags: irqflags);
658
659 return 0;
660}
661
662static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
663{
664 unsigned long irqflags;
665 struct switchtec_ntb *sndev = ntb_sndev(ntb);
666
667 if (db_bits & ~sndev->db_valid_mask)
668 return -EINVAL;
669
670 spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
671
672 sndev->db_mask &= ~(db_bits << sndev->db_shift);
673 iowrite64(val: ~sndev->db_mask, addr: &sndev->mmio_self_dbmsg->idb_mask);
674
675 spin_unlock_irqrestore(lock: &sndev->db_mask_lock, flags: irqflags);
676
677 return 0;
678}
679
680static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb)
681{
682 struct switchtec_ntb *sndev = ntb_sndev(ntb);
683
684 return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask;
685}
686
687static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb,
688 phys_addr_t *db_addr,
689 resource_size_t *db_size,
690 u64 *db_data,
691 int db_bit)
692{
693 struct switchtec_ntb *sndev = ntb_sndev(ntb);
694 unsigned long offset;
695
696 if (unlikely(db_bit >= BITS_PER_LONG_LONG))
697 return -EINVAL;
698
699 offset = (unsigned long)sndev->mmio_peer_dbmsg->odb -
700 (unsigned long)sndev->stdev->mmio;
701
702 offset += sndev->db_shift / 8;
703
704 if (db_addr)
705 *db_addr = pci_resource_start(ntb->pdev, 0) + offset;
706 if (db_size)
707 *db_size = sizeof(u32);
708 if (db_data)
709 *db_data = BIT_ULL(db_bit) << sndev->db_peer_shift;
710
711 return 0;
712}
713
714static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
715{
716 struct switchtec_ntb *sndev = ntb_sndev(ntb);
717
718 iowrite64(val: db_bits << sndev->db_peer_shift,
719 addr: &sndev->mmio_peer_dbmsg->odb);
720
721 return 0;
722}
723
724static int switchtec_ntb_spad_count(struct ntb_dev *ntb)
725{
726 struct switchtec_ntb *sndev = ntb_sndev(ntb);
727
728 return ARRAY_SIZE(sndev->self_shared->spad);
729}
730
731static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx)
732{
733 struct switchtec_ntb *sndev = ntb_sndev(ntb);
734
735 if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
736 return 0;
737
738 if (!sndev->self_shared)
739 return 0;
740
741 return sndev->self_shared->spad[idx];
742}
743
744static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
745{
746 struct switchtec_ntb *sndev = ntb_sndev(ntb);
747
748 if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
749 return -EINVAL;
750
751 if (!sndev->self_shared)
752 return -EIO;
753
754 sndev->self_shared->spad[idx] = val;
755
756 return 0;
757}
758
759static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx,
760 int sidx)
761{
762 struct switchtec_ntb *sndev = ntb_sndev(ntb);
763
764 if (pidx != NTB_DEF_PEER_IDX)
765 return -EINVAL;
766
767 if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
768 return 0;
769
770 if (!sndev->peer_shared)
771 return 0;
772
773 return ioread32(&sndev->peer_shared->spad[sidx]);
774}
775
776static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
777 int sidx, u32 val)
778{
779 struct switchtec_ntb *sndev = ntb_sndev(ntb);
780
781 if (pidx != NTB_DEF_PEER_IDX)
782 return -EINVAL;
783
784 if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
785 return -EINVAL;
786
787 if (!sndev->peer_shared)
788 return -EIO;
789
790 iowrite32(val, &sndev->peer_shared->spad[sidx]);
791
792 return 0;
793}
794
795static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx,
796 int sidx, phys_addr_t *spad_addr)
797{
798 struct switchtec_ntb *sndev = ntb_sndev(ntb);
799 unsigned long offset;
800
801 if (pidx != NTB_DEF_PEER_IDX)
802 return -EINVAL;
803
804 offset = (unsigned long)&sndev->peer_shared->spad[sidx] -
805 (unsigned long)sndev->stdev->mmio;
806
807 if (spad_addr)
808 *spad_addr = pci_resource_start(ntb->pdev, 0) + offset;
809
810 return 0;
811}
812
813static const struct ntb_dev_ops switchtec_ntb_ops = {
814 .mw_count = switchtec_ntb_mw_count,
815 .mw_get_align = switchtec_ntb_mw_get_align,
816 .mw_set_trans = switchtec_ntb_mw_set_trans,
817 .peer_mw_count = switchtec_ntb_peer_mw_count,
818 .peer_mw_get_addr = switchtec_ntb_peer_mw_get_addr,
819 .link_is_up = switchtec_ntb_link_is_up,
820 .link_enable = switchtec_ntb_link_enable,
821 .link_disable = switchtec_ntb_link_disable,
822 .db_valid_mask = switchtec_ntb_db_valid_mask,
823 .db_vector_count = switchtec_ntb_db_vector_count,
824 .db_vector_mask = switchtec_ntb_db_vector_mask,
825 .db_read = switchtec_ntb_db_read,
826 .db_clear = switchtec_ntb_db_clear,
827 .db_set_mask = switchtec_ntb_db_set_mask,
828 .db_clear_mask = switchtec_ntb_db_clear_mask,
829 .db_read_mask = switchtec_ntb_db_read_mask,
830 .peer_db_addr = switchtec_ntb_peer_db_addr,
831 .peer_db_set = switchtec_ntb_peer_db_set,
832 .spad_count = switchtec_ntb_spad_count,
833 .spad_read = switchtec_ntb_spad_read,
834 .spad_write = switchtec_ntb_spad_write,
835 .peer_spad_read = switchtec_ntb_peer_spad_read,
836 .peer_spad_write = switchtec_ntb_peer_spad_write,
837 .peer_spad_addr = switchtec_ntb_peer_spad_addr,
838};
839
840static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
841{
842 u64 tpart_vec;
843 int self;
844 u64 part_map;
845
846 sndev->ntb.pdev = sndev->stdev->pdev;
847 sndev->ntb.topo = NTB_TOPO_SWITCH;
848 sndev->ntb.ops = &switchtec_ntb_ops;
849
850 INIT_WORK(&sndev->check_link_status_work, check_link_status_work);
851 sndev->link_force_down = false;
852
853 sndev->self_partition = sndev->stdev->partition;
854
855 sndev->mmio_ntb = sndev->stdev->mmio_ntb;
856
857 self = sndev->self_partition;
858 tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high);
859 tpart_vec <<= 32;
860 tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low);
861
862 part_map = ioread64(addr: &sndev->mmio_ntb->ep_map);
863 tpart_vec &= part_map;
864 part_map &= ~(1 << sndev->self_partition);
865
866 if (!tpart_vec) {
867 if (sndev->stdev->partition_count != 2) {
868 dev_err(&sndev->stdev->dev,
869 "ntb target partition not defined\n");
870 return -ENODEV;
871 }
872
873 if (!part_map) {
874 dev_err(&sndev->stdev->dev,
875 "peer partition is not NT partition\n");
876 return -ENODEV;
877 }
878
879 sndev->peer_partition = __ffs64(word: part_map);
880 } else {
881 if (__ffs64(word: tpart_vec) != (fls64(x: tpart_vec) - 1)) {
882 dev_err(&sndev->stdev->dev,
883 "ntb driver only supports 1 pair of 1-1 ntb mapping\n");
884 return -ENODEV;
885 }
886
887 sndev->peer_partition = __ffs64(word: tpart_vec);
888 if (!(part_map & (1ULL << sndev->peer_partition))) {
889 dev_err(&sndev->stdev->dev,
890 "ntb target partition is not NT partition\n");
891 return -ENODEV;
892 }
893 }
894
895 dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n",
896 sndev->self_partition, sndev->stdev->partition_count);
897
898 sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb +
899 SWITCHTEC_NTB_REG_CTRL_OFFSET;
900 sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb +
901 SWITCHTEC_NTB_REG_DBMSG_OFFSET;
902
903 sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition];
904 sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition];
905 sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition];
906 sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg;
907
908 return 0;
909}
910
911static int config_rsvd_lut_win(struct switchtec_ntb *sndev,
912 struct ntb_ctrl_regs __iomem *ctl,
913 int lut_idx, int partition, u64 addr)
914{
915 int peer_bar = sndev->peer_direct_mw_to_bar[0];
916 u32 ctl_val;
917 int rc;
918
919 rc = switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_LOCK,
920 wait_status: NTB_CTRL_PART_STATUS_LOCKED);
921 if (rc)
922 return rc;
923
924 ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl);
925 ctl_val &= 0xFF;
926 ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN;
927 ctl_val |= ilog2(LUT_SIZE) << 8;
928 ctl_val |= (sndev->nr_lut_mw - 1) << 14;
929 iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl);
930
931 iowrite64(val: (NTB_CTRL_LUT_EN | (partition << 1) | addr),
932 addr: &ctl->lut_entry[lut_idx]);
933
934 rc = switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_CFG,
935 wait_status: NTB_CTRL_PART_STATUS_NORMAL);
936 if (rc) {
937 u32 bar_error, lut_error;
938
939 bar_error = ioread32(&ctl->bar_error);
940 lut_error = ioread32(&ctl->lut_error);
941 dev_err(&sndev->stdev->dev,
942 "Error setting up reserved lut window: %08x / %08x\n",
943 bar_error, lut_error);
944 return rc;
945 }
946
947 return 0;
948}
949
950static int config_req_id_table(struct switchtec_ntb *sndev,
951 struct ntb_ctrl_regs __iomem *mmio_ctrl,
952 int *req_ids, int count)
953{
954 int i, rc = 0;
955 u32 error;
956 u32 proxy_id;
957
958 if (ioread16(&mmio_ctrl->req_id_table_size) < count) {
959 dev_err(&sndev->stdev->dev,
960 "Not enough requester IDs available.\n");
961 return -EFAULT;
962 }
963
964 rc = switchtec_ntb_part_op(sndev, ctl: mmio_ctrl,
965 op: NTB_CTRL_PART_OP_LOCK,
966 wait_status: NTB_CTRL_PART_STATUS_LOCKED);
967 if (rc)
968 return rc;
969
970 for (i = 0; i < count; i++) {
971 iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN,
972 &mmio_ctrl->req_id_table[i]);
973
974 proxy_id = ioread32(&mmio_ctrl->req_id_table[i]);
975 dev_dbg(&sndev->stdev->dev,
976 "Requester ID %02X:%02X.%X -> BB:%02X.%X\n",
977 req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F,
978 req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F,
979 (proxy_id >> 1) & 0x7);
980 }
981
982 rc = switchtec_ntb_part_op(sndev, ctl: mmio_ctrl,
983 op: NTB_CTRL_PART_OP_CFG,
984 wait_status: NTB_CTRL_PART_STATUS_NORMAL);
985
986 if (rc == -EIO) {
987 error = ioread32(&mmio_ctrl->req_id_error);
988 dev_err(&sndev->stdev->dev,
989 "Error setting up the requester ID table: %08x\n",
990 error);
991 }
992
993 return 0;
994}
995
996static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx,
997 u64 *mw_addrs, int mw_count)
998{
999 int rc, i;
1000 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl;
1001 u64 addr;
1002 size_t size, offset;
1003 int bar;
1004 int xlate_pos;
1005 u32 ctl_val;
1006
1007 rc = switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_LOCK,
1008 wait_status: NTB_CTRL_PART_STATUS_LOCKED);
1009 if (rc)
1010 return rc;
1011
1012 for (i = 0; i < sndev->nr_lut_mw; i++) {
1013 if (i == ntb_lut_idx)
1014 continue;
1015
1016 addr = mw_addrs[0] + LUT_SIZE * i;
1017
1018 iowrite64(val: (NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) |
1019 addr),
1020 addr: &ctl->lut_entry[i]);
1021 }
1022
1023 sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count);
1024
1025 for (i = 0; i < sndev->nr_direct_mw; i++) {
1026 bar = sndev->direct_mw_to_bar[i];
1027 offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0;
1028 addr = mw_addrs[i] + offset;
1029 size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
1030 xlate_pos = ilog2(size);
1031
1032 if (offset && size > offset)
1033 size = offset;
1034
1035 ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
1036 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
1037
1038 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
1039 iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
1040 &ctl->bar_entry[bar].win_size);
1041 iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
1042 iowrite64(val: sndev->peer_partition | addr,
1043 addr: &ctl->bar_entry[bar].xlate_addr);
1044 }
1045
1046 rc = switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_CFG,
1047 wait_status: NTB_CTRL_PART_STATUS_NORMAL);
1048 if (rc) {
1049 u32 bar_error, lut_error;
1050
1051 bar_error = ioread32(&ctl->bar_error);
1052 lut_error = ioread32(&ctl->lut_error);
1053 dev_err(&sndev->stdev->dev,
1054 "Error setting up cross link windows: %08x / %08x\n",
1055 bar_error, lut_error);
1056 return rc;
1057 }
1058
1059 return 0;
1060}
1061
1062static int crosslink_setup_req_ids(struct switchtec_ntb *sndev,
1063 struct ntb_ctrl_regs __iomem *mmio_ctrl)
1064{
1065 int req_ids[16];
1066 int i;
1067 u32 proxy_id;
1068
1069 for (i = 0; i < ARRAY_SIZE(req_ids); i++) {
1070 proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]);
1071
1072 if (!(proxy_id & NTB_CTRL_REQ_ID_EN))
1073 break;
1074
1075 req_ids[i] = ((proxy_id >> 1) & 0xFF);
1076 }
1077
1078 return config_req_id_table(sndev, mmio_ctrl, req_ids, count: i);
1079}
1080
1081/*
1082 * In crosslink configuration there is a virtual partition in the
1083 * middle of the two switches. The BARs in this partition have to be
1084 * enumerated and assigned addresses.
1085 */
1086static int crosslink_enum_partition(struct switchtec_ntb *sndev,
1087 u64 *bar_addrs)
1088{
1089 struct part_cfg_regs __iomem *part_cfg =
1090 &sndev->stdev->mmio_part_cfg_all[sndev->peer_partition];
1091 u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF;
1092 struct pff_csr_regs __iomem *mmio_pff =
1093 &sndev->stdev->mmio_pff_csr[pff];
1094 const u64 bar_space = 0x1000000000LL;
1095 u64 bar_addr;
1096 int bar_cnt = 0;
1097 int i;
1098
1099 iowrite16(0x6, &mmio_pff->pcicmd);
1100
1101 for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) {
1102 iowrite64(val: bar_space * i, addr: &mmio_pff->pci_bar64[i]);
1103 bar_addr = ioread64(addr: &mmio_pff->pci_bar64[i]);
1104 bar_addr &= ~0xf;
1105
1106 dev_dbg(&sndev->stdev->dev,
1107 "Crosslink BAR%d addr: %llx\n",
1108 i*2, bar_addr);
1109
1110 if (bar_addr != bar_space * i)
1111 continue;
1112
1113 bar_addrs[bar_cnt++] = bar_addr;
1114 }
1115
1116 return bar_cnt;
1117}
1118
1119static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev)
1120{
1121 int rc;
1122 int bar = sndev->direct_mw_to_bar[0];
1123 const int ntb_lut_idx = 1;
1124 u64 bar_addrs[6];
1125 u64 addr;
1126 int offset;
1127 int bar_cnt;
1128
1129 if (!crosslink_is_enabled(sndev))
1130 return 0;
1131
1132 dev_info(&sndev->stdev->dev, "Using crosslink configuration\n");
1133 sndev->ntb.topo = NTB_TOPO_CROSSLINK;
1134
1135 bar_cnt = crosslink_enum_partition(sndev, bar_addrs);
1136 if (bar_cnt < sndev->nr_direct_mw + 1) {
1137 dev_err(&sndev->stdev->dev,
1138 "Error enumerating crosslink partition\n");
1139 return -EINVAL;
1140 }
1141
1142 addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET +
1143 SWITCHTEC_NTB_REG_DBMSG_OFFSET +
1144 sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition);
1145
1146 offset = addr & (LUT_SIZE - 1);
1147 addr -= offset;
1148
1149 rc = config_rsvd_lut_win(sndev, ctl: sndev->mmio_self_ctrl, lut_idx: ntb_lut_idx,
1150 partition: sndev->peer_partition, addr);
1151 if (rc)
1152 return rc;
1153
1154 rc = crosslink_setup_mws(sndev, ntb_lut_idx, mw_addrs: &bar_addrs[1],
1155 mw_count: bar_cnt - 1);
1156 if (rc)
1157 return rc;
1158
1159 rc = crosslink_setup_req_ids(sndev, mmio_ctrl: sndev->mmio_peer_ctrl);
1160 if (rc)
1161 return rc;
1162
1163 sndev->mmio_xlink_win = pci_iomap_range(dev: sndev->stdev->pdev, bar,
1164 LUT_SIZE, LUT_SIZE);
1165 if (!sndev->mmio_xlink_win) {
1166 rc = -ENOMEM;
1167 return rc;
1168 }
1169
1170 sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset;
1171 sndev->nr_rsvd_luts++;
1172
1173 crosslink_init_dbmsgs(sndev);
1174
1175 return 0;
1176}
1177
1178static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev)
1179{
1180 if (sndev->mmio_xlink_win)
1181 pci_iounmap(dev: sndev->stdev->pdev, sndev->mmio_xlink_win);
1182}
1183
1184static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl)
1185{
1186 int i;
1187 int cnt = 0;
1188
1189 for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) {
1190 u32 r = ioread32(&ctrl->bar_entry[i].ctl);
1191
1192 if (r & NTB_CTRL_BAR_VALID)
1193 map[cnt++] = i;
1194 }
1195
1196 return cnt;
1197}
1198
1199static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
1200{
1201 sndev->nr_direct_mw = map_bars(map: sndev->direct_mw_to_bar,
1202 ctrl: sndev->mmio_self_ctrl);
1203
1204 sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries);
1205 sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw);
1206
1207 dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n",
1208 sndev->nr_direct_mw, sndev->nr_lut_mw);
1209
1210 sndev->peer_nr_direct_mw = map_bars(map: sndev->peer_direct_mw_to_bar,
1211 ctrl: sndev->mmio_peer_ctrl);
1212
1213 sndev->peer_nr_lut_mw =
1214 ioread16(&sndev->mmio_peer_ctrl->lut_table_entries);
1215 sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw);
1216
1217 dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n",
1218 sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw);
1219
1220}
1221
1222/*
1223 * There are 64 doorbells in the switch hardware but this is
1224 * shared among all partitions. So we must split them in half
1225 * (32 for each partition). However, the message interrupts are
1226 * also shared with the top 4 doorbells so we just limit this to
1227 * 28 doorbells per partition.
1228 *
1229 * In crosslink mode, each side has it's own dbmsg register so
1230 * they can each use all 60 of the available doorbells.
1231 */
1232static void switchtec_ntb_init_db(struct switchtec_ntb *sndev)
1233{
1234 sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL;
1235
1236 if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) {
1237 sndev->db_shift = 0;
1238 sndev->db_peer_shift = 0;
1239 sndev->db_valid_mask = sndev->db_mask;
1240 } else if (sndev->self_partition < sndev->peer_partition) {
1241 sndev->db_shift = 0;
1242 sndev->db_peer_shift = 32;
1243 sndev->db_valid_mask = 0x0FFFFFFF;
1244 } else {
1245 sndev->db_shift = 32;
1246 sndev->db_peer_shift = 0;
1247 sndev->db_valid_mask = 0x0FFFFFFF;
1248 }
1249
1250 iowrite64(val: ~sndev->db_mask, addr: &sndev->mmio_self_dbmsg->idb_mask);
1251 iowrite64(val: sndev->db_valid_mask << sndev->db_peer_shift,
1252 addr: &sndev->mmio_peer_dbmsg->odb_mask);
1253
1254 dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n",
1255 sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask);
1256}
1257
1258static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev)
1259{
1260 int i;
1261 u32 msg_map = 0;
1262
1263 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1264 int m = i | sndev->peer_partition << 2;
1265
1266 msg_map |= m << i * 8;
1267 }
1268
1269 iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map);
1270
1271 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++)
1272 iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK,
1273 addr: &sndev->mmio_self_dbmsg->imsg[i]);
1274}
1275
1276static int
1277switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev)
1278{
1279 int req_ids[2];
1280
1281 /*
1282 * Root Complex Requester ID (which is 0:00.0)
1283 */
1284 req_ids[0] = 0;
1285
1286 /*
1287 * Host Bridge Requester ID (as read from the mmap address)
1288 */
1289 req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id);
1290
1291 return config_req_id_table(sndev, mmio_ctrl: sndev->mmio_self_ctrl, req_ids,
1292 ARRAY_SIZE(req_ids));
1293}
1294
1295static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)
1296{
1297 int i;
1298
1299 memset(sndev->self_shared, 0, LUT_SIZE);
1300 sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC;
1301 sndev->self_shared->partition_id = sndev->stdev->partition;
1302
1303 for (i = 0; i < sndev->nr_direct_mw; i++) {
1304 int bar = sndev->direct_mw_to_bar[i];
1305 resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar);
1306
1307 if (i == 0)
1308 sz = min_t(resource_size_t, sz,
1309 LUT_SIZE * sndev->nr_lut_mw);
1310
1311 sndev->self_shared->mw_sizes[i] = sz;
1312 }
1313
1314 for (i = 0; i < sndev->nr_lut_mw; i++) {
1315 int idx = sndev->nr_direct_mw + i;
1316
1317 sndev->self_shared->mw_sizes[idx] = LUT_SIZE;
1318 }
1319}
1320
1321static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev)
1322{
1323 int self_bar = sndev->direct_mw_to_bar[0];
1324 int rc;
1325
1326 sndev->nr_rsvd_luts++;
1327 sndev->self_shared = dma_alloc_coherent(dev: &sndev->stdev->pdev->dev,
1328 LUT_SIZE,
1329 dma_handle: &sndev->self_shared_dma,
1330 GFP_KERNEL);
1331 if (!sndev->self_shared) {
1332 dev_err(&sndev->stdev->dev,
1333 "unable to allocate memory for shared mw\n");
1334 return -ENOMEM;
1335 }
1336
1337 switchtec_ntb_init_shared(sndev);
1338
1339 rc = config_rsvd_lut_win(sndev, ctl: sndev->mmio_peer_ctrl, lut_idx: 0,
1340 partition: sndev->self_partition,
1341 addr: sndev->self_shared_dma);
1342 if (rc)
1343 goto unalloc_and_exit;
1344
1345 sndev->peer_shared = pci_iomap(dev: sndev->stdev->pdev, bar: self_bar, LUT_SIZE);
1346 if (!sndev->peer_shared) {
1347 rc = -ENOMEM;
1348 goto unalloc_and_exit;
1349 }
1350
1351 dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n");
1352 return 0;
1353
1354unalloc_and_exit:
1355 dma_free_coherent(dev: &sndev->stdev->pdev->dev, LUT_SIZE,
1356 cpu_addr: sndev->self_shared, dma_handle: sndev->self_shared_dma);
1357
1358 return rc;
1359}
1360
1361static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev)
1362{
1363 if (sndev->peer_shared)
1364 pci_iounmap(dev: sndev->stdev->pdev, sndev->peer_shared);
1365
1366 if (sndev->self_shared)
1367 dma_free_coherent(dev: &sndev->stdev->pdev->dev, LUT_SIZE,
1368 cpu_addr: sndev->self_shared,
1369 dma_handle: sndev->self_shared_dma);
1370 sndev->nr_rsvd_luts--;
1371}
1372
1373static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev)
1374{
1375 struct switchtec_ntb *sndev = dev;
1376
1377 dev_dbg(&sndev->stdev->dev, "doorbell\n");
1378
1379 ntb_db_event(ntb: &sndev->ntb, vector: 0);
1380
1381 return IRQ_HANDLED;
1382}
1383
1384static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev)
1385{
1386 int i;
1387 struct switchtec_ntb *sndev = dev;
1388
1389 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1390 u64 msg = ioread64(addr: &sndev->mmio_self_dbmsg->imsg[i]);
1391
1392 if (msg & NTB_DBMSG_IMSG_STATUS) {
1393 dev_dbg(&sndev->stdev->dev, "message: %d %08x\n",
1394 i, (u32)msg);
1395 iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status);
1396
1397 if (i == LINK_MESSAGE)
1398 switchtec_ntb_check_link(sndev, msg);
1399 }
1400 }
1401
1402 return IRQ_HANDLED;
1403}
1404
1405static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev)
1406{
1407 int i;
1408 int rc;
1409 int doorbell_irq = 0;
1410 int message_irq = 0;
1411 int event_irq;
1412 int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map);
1413
1414 event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number);
1415
1416 while (doorbell_irq == event_irq)
1417 doorbell_irq++;
1418 while (message_irq == doorbell_irq ||
1419 message_irq == event_irq)
1420 message_irq++;
1421
1422 dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n",
1423 event_irq, doorbell_irq, message_irq);
1424
1425 for (i = 0; i < idb_vecs - 4; i++)
1426 iowrite8(doorbell_irq,
1427 &sndev->mmio_self_dbmsg->idb_vec_map[i]);
1428
1429 for (; i < idb_vecs; i++)
1430 iowrite8(message_irq,
1431 &sndev->mmio_self_dbmsg->idb_vec_map[i]);
1432
1433 sndev->doorbell_irq = pci_irq_vector(dev: sndev->stdev->pdev, nr: doorbell_irq);
1434 sndev->message_irq = pci_irq_vector(dev: sndev->stdev->pdev, nr: message_irq);
1435
1436 rc = request_irq(irq: sndev->doorbell_irq,
1437 handler: switchtec_ntb_doorbell_isr, flags: 0,
1438 name: "switchtec_ntb_doorbell", dev: sndev);
1439 if (rc)
1440 return rc;
1441
1442 rc = request_irq(irq: sndev->message_irq,
1443 handler: switchtec_ntb_message_isr, flags: 0,
1444 name: "switchtec_ntb_message", dev: sndev);
1445 if (rc) {
1446 free_irq(sndev->doorbell_irq, sndev);
1447 return rc;
1448 }
1449
1450 return 0;
1451}
1452
1453static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev)
1454{
1455 free_irq(sndev->doorbell_irq, sndev);
1456 free_irq(sndev->message_irq, sndev);
1457}
1458
1459static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev)
1460{
1461 int rc;
1462
1463 if (crosslink_is_enabled(sndev))
1464 return 0;
1465
1466 dev_info(&sndev->stdev->dev, "reinitialize shared memory window\n");
1467 rc = config_rsvd_lut_win(sndev, ctl: sndev->mmio_peer_ctrl, lut_idx: 0,
1468 partition: sndev->self_partition,
1469 addr: sndev->self_shared_dma);
1470 return rc;
1471}
1472
1473static int switchtec_ntb_add(struct device *dev)
1474{
1475 struct switchtec_dev *stdev = to_stdev(dev);
1476 struct switchtec_ntb *sndev;
1477 int rc;
1478
1479 stdev->sndev = NULL;
1480
1481 if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8))
1482 return -ENODEV;
1483
1484 sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev));
1485 if (!sndev)
1486 return -ENOMEM;
1487
1488 sndev->stdev = stdev;
1489 rc = switchtec_ntb_init_sndev(sndev);
1490 if (rc)
1491 goto free_and_exit;
1492
1493 switchtec_ntb_init_mw(sndev);
1494
1495 rc = switchtec_ntb_init_req_id_table(sndev);
1496 if (rc)
1497 goto free_and_exit;
1498
1499 rc = switchtec_ntb_init_crosslink(sndev);
1500 if (rc)
1501 goto free_and_exit;
1502
1503 switchtec_ntb_init_db(sndev);
1504 switchtec_ntb_init_msgs(sndev);
1505
1506 rc = switchtec_ntb_init_shared_mw(sndev);
1507 if (rc)
1508 goto deinit_crosslink;
1509
1510 rc = switchtec_ntb_init_db_msg_irq(sndev);
1511 if (rc)
1512 goto deinit_shared_and_exit;
1513
1514 /*
1515 * If this host crashed, the other host may think the link is
1516 * still up. Tell them to force it down (it will go back up
1517 * once we register the ntb device).
1518 */
1519 switchtec_ntb_send_msg(sndev, idx: LINK_MESSAGE, val: MSG_LINK_FORCE_DOWN);
1520
1521 rc = ntb_register_device(ntb: &sndev->ntb);
1522 if (rc)
1523 goto deinit_and_exit;
1524
1525 stdev->sndev = sndev;
1526 stdev->link_notifier = switchtec_ntb_link_notification;
1527 dev_info(dev, "NTB device registered\n");
1528
1529 return 0;
1530
1531deinit_and_exit:
1532 switchtec_ntb_deinit_db_msg_irq(sndev);
1533deinit_shared_and_exit:
1534 switchtec_ntb_deinit_shared_mw(sndev);
1535deinit_crosslink:
1536 switchtec_ntb_deinit_crosslink(sndev);
1537free_and_exit:
1538 kfree(objp: sndev);
1539 dev_err(dev, "failed to register ntb device: %d\n", rc);
1540 return rc;
1541}
1542
1543static void switchtec_ntb_remove(struct device *dev)
1544{
1545 struct switchtec_dev *stdev = to_stdev(dev);
1546 struct switchtec_ntb *sndev = stdev->sndev;
1547
1548 if (!sndev)
1549 return;
1550
1551 stdev->link_notifier = NULL;
1552 stdev->sndev = NULL;
1553 ntb_unregister_device(ntb: &sndev->ntb);
1554 switchtec_ntb_deinit_db_msg_irq(sndev);
1555 switchtec_ntb_deinit_shared_mw(sndev);
1556 switchtec_ntb_deinit_crosslink(sndev);
1557 cancel_work_sync(work: &sndev->check_link_status_work);
1558 kfree(objp: sndev);
1559 dev_info(dev, "ntb device unregistered\n");
1560}
1561
1562static struct class_interface switchtec_interface = {
1563 .add_dev = switchtec_ntb_add,
1564 .remove_dev = switchtec_ntb_remove,
1565};
1566
1567static int __init switchtec_ntb_init(void)
1568{
1569 switchtec_interface.class = &switchtec_class;
1570 return class_interface_register(&switchtec_interface);
1571}
1572module_init(switchtec_ntb_init);
1573
1574static void __exit switchtec_ntb_exit(void)
1575{
1576 class_interface_unregister(&switchtec_interface);
1577}
1578module_exit(switchtec_ntb_exit);
1579

source code of linux/drivers/ntb/hw/mscc/ntb_hw_switchtec.c