1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2
3#include <linux/bitmap.h>
4#include <linux/delay.h>
5#include <linux/interrupt.h>
6#include <linux/io.h>
7#include <linux/io-64-nonatomic-lo-hi.h>
8#include <linux/mm.h>
9#include <linux/module.h>
10#include <linux/nvme.h>
11#include <linux/pci.h>
12#include <linux/wait.h>
13#include <linux/sched/signal.h>
14
15#include "fun_queue.h"
16#include "fun_dev.h"
17
18#define FUN_ADMIN_CMD_TO_MS 3000
19
20enum {
21 AQA_ASQS_SHIFT = 0,
22 AQA_ACQS_SHIFT = 16,
23 AQA_MIN_QUEUE_SIZE = 2,
24 AQA_MAX_QUEUE_SIZE = 4096
25};
26
27/* context for admin commands */
28struct fun_cmd_ctx {
29 fun_admin_callback_t cb; /* callback to invoke on completion */
30 void *cb_data; /* user data provided to callback */
31 int cpu; /* CPU where the cmd's tag was allocated */
32};
33
34/* Context for synchronous admin commands. */
35struct fun_sync_cmd_ctx {
36 struct completion compl;
37 u8 *rsp_buf; /* caller provided response buffer */
38 unsigned int rsp_len; /* response buffer size */
39 u8 rsp_status; /* command response status */
40};
41
42/* Wait for the CSTS.RDY bit to match @enabled. */
43static int fun_wait_ready(struct fun_dev *fdev, bool enabled)
44{
45 unsigned int cap_to = NVME_CAP_TIMEOUT(fdev->cap_reg);
46 u32 bit = enabled ? NVME_CSTS_RDY : 0;
47 unsigned long deadline;
48
49 deadline = ((cap_to + 1) * HZ / 2) + jiffies; /* CAP.TO is in 500ms */
50
51 for (;;) {
52 u32 csts = readl(addr: fdev->bar + NVME_REG_CSTS);
53
54 if (csts == ~0) {
55 dev_err(fdev->dev, "CSTS register read %#x\n", csts);
56 return -EIO;
57 }
58
59 if ((csts & NVME_CSTS_RDY) == bit)
60 return 0;
61
62 if (time_is_before_jiffies(deadline))
63 break;
64
65 msleep(msecs: 100);
66 }
67
68 dev_err(fdev->dev,
69 "Timed out waiting for device to indicate RDY %u; aborting %s\n",
70 enabled, enabled ? "initialization" : "reset");
71 return -ETIMEDOUT;
72}
73
74/* Check CSTS and return an error if it is unreadable or has unexpected
75 * RDY value.
76 */
77static int fun_check_csts_rdy(struct fun_dev *fdev, unsigned int expected_rdy)
78{
79 u32 csts = readl(addr: fdev->bar + NVME_REG_CSTS);
80 u32 actual_rdy = csts & NVME_CSTS_RDY;
81
82 if (csts == ~0) {
83 dev_err(fdev->dev, "CSTS register read %#x\n", csts);
84 return -EIO;
85 }
86 if (actual_rdy != expected_rdy) {
87 dev_err(fdev->dev, "Unexpected CSTS RDY %u\n", actual_rdy);
88 return -EINVAL;
89 }
90 return 0;
91}
92
93/* Check that CSTS RDY has the expected value. Then write a new value to the CC
94 * register and wait for CSTS RDY to match the new CC ENABLE state.
95 */
96static int fun_update_cc_enable(struct fun_dev *fdev, unsigned int initial_rdy)
97{
98 int rc = fun_check_csts_rdy(fdev, expected_rdy: initial_rdy);
99
100 if (rc)
101 return rc;
102 writel(val: fdev->cc_reg, addr: fdev->bar + NVME_REG_CC);
103 return fun_wait_ready(fdev, enabled: !!(fdev->cc_reg & NVME_CC_ENABLE));
104}
105
106static int fun_disable_ctrl(struct fun_dev *fdev)
107{
108 fdev->cc_reg &= ~(NVME_CC_SHN_MASK | NVME_CC_ENABLE);
109 return fun_update_cc_enable(fdev, initial_rdy: 1);
110}
111
112static int fun_enable_ctrl(struct fun_dev *fdev, u32 admin_cqesz_log2,
113 u32 admin_sqesz_log2)
114{
115 fdev->cc_reg = (admin_cqesz_log2 << NVME_CC_IOCQES_SHIFT) |
116 (admin_sqesz_log2 << NVME_CC_IOSQES_SHIFT) |
117 ((PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT) |
118 NVME_CC_ENABLE;
119
120 return fun_update_cc_enable(fdev, initial_rdy: 0);
121}
122
123static int fun_map_bars(struct fun_dev *fdev, const char *name)
124{
125 struct pci_dev *pdev = to_pci_dev(fdev->dev);
126 int err;
127
128 err = pci_request_mem_regions(pdev, name);
129 if (err) {
130 dev_err(&pdev->dev,
131 "Couldn't get PCI memory resources, err %d\n", err);
132 return err;
133 }
134
135 fdev->bar = pci_ioremap_bar(pdev, bar: 0);
136 if (!fdev->bar) {
137 dev_err(&pdev->dev, "Couldn't map BAR 0\n");
138 pci_release_mem_regions(pdev);
139 return -ENOMEM;
140 }
141
142 return 0;
143}
144
145static void fun_unmap_bars(struct fun_dev *fdev)
146{
147 struct pci_dev *pdev = to_pci_dev(fdev->dev);
148
149 if (fdev->bar) {
150 iounmap(addr: fdev->bar);
151 fdev->bar = NULL;
152 pci_release_mem_regions(pdev);
153 }
154}
155
156static int fun_set_dma_masks(struct device *dev)
157{
158 int err;
159
160 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
161 if (err)
162 dev_err(dev, "DMA mask configuration failed, err %d\n", err);
163 return err;
164}
165
166static irqreturn_t fun_admin_irq(int irq, void *data)
167{
168 struct fun_queue *funq = data;
169
170 return fun_process_cq(funq, max: 0) ? IRQ_HANDLED : IRQ_NONE;
171}
172
173static void fun_complete_admin_cmd(struct fun_queue *funq, void *data,
174 void *entry, const struct fun_cqe_info *info)
175{
176 const struct fun_admin_rsp_common *rsp_common = entry;
177 struct fun_dev *fdev = funq->fdev;
178 struct fun_cmd_ctx *cmd_ctx;
179 int cpu;
180 u16 cid;
181
182 if (info->sqhd == cpu_to_be16(0xffff)) {
183 dev_dbg(fdev->dev, "adminq event");
184 if (fdev->adminq_cb)
185 fdev->adminq_cb(fdev, entry);
186 return;
187 }
188
189 cid = be16_to_cpu(rsp_common->cid);
190 dev_dbg(fdev->dev, "admin CQE cid %u, op %u, ret %u\n", cid,
191 rsp_common->op, rsp_common->ret);
192
193 cmd_ctx = &fdev->cmd_ctx[cid];
194 if (cmd_ctx->cpu < 0) {
195 dev_err(fdev->dev,
196 "admin CQE with CID=%u, op=%u does not match a pending command\n",
197 cid, rsp_common->op);
198 return;
199 }
200
201 if (cmd_ctx->cb)
202 cmd_ctx->cb(fdev, entry, xchg(&cmd_ctx->cb_data, NULL));
203
204 cpu = cmd_ctx->cpu;
205 cmd_ctx->cpu = -1;
206 sbitmap_queue_clear(sbq: &fdev->admin_sbq, nr: cid, cpu);
207}
208
209static int fun_init_cmd_ctx(struct fun_dev *fdev, unsigned int ntags)
210{
211 unsigned int i;
212
213 fdev->cmd_ctx = kvcalloc(ntags, sizeof(*fdev->cmd_ctx), GFP_KERNEL);
214 if (!fdev->cmd_ctx)
215 return -ENOMEM;
216
217 for (i = 0; i < ntags; i++)
218 fdev->cmd_ctx[i].cpu = -1;
219
220 return 0;
221}
222
223/* Allocate and enable an admin queue and assign it the first IRQ vector. */
224static int fun_enable_admin_queue(struct fun_dev *fdev,
225 const struct fun_dev_params *areq)
226{
227 struct fun_queue_alloc_req qreq = {
228 .cqe_size_log2 = areq->cqe_size_log2,
229 .sqe_size_log2 = areq->sqe_size_log2,
230 .cq_depth = areq->cq_depth,
231 .sq_depth = areq->sq_depth,
232 .rq_depth = areq->rq_depth,
233 };
234 unsigned int ntags = areq->sq_depth - 1;
235 struct fun_queue *funq;
236 int rc;
237
238 if (fdev->admin_q)
239 return -EEXIST;
240
241 if (areq->sq_depth < AQA_MIN_QUEUE_SIZE ||
242 areq->sq_depth > AQA_MAX_QUEUE_SIZE ||
243 areq->cq_depth < AQA_MIN_QUEUE_SIZE ||
244 areq->cq_depth > AQA_MAX_QUEUE_SIZE)
245 return -EINVAL;
246
247 fdev->admin_q = fun_alloc_queue(fdev, qid: 0, req: &qreq);
248 if (!fdev->admin_q)
249 return -ENOMEM;
250
251 rc = fun_init_cmd_ctx(fdev, ntags);
252 if (rc)
253 goto free_q;
254
255 rc = sbitmap_queue_init_node(sbq: &fdev->admin_sbq, depth: ntags, shift: -1, round_robin: false,
256 GFP_KERNEL, node: dev_to_node(dev: fdev->dev));
257 if (rc)
258 goto free_cmd_ctx;
259
260 funq = fdev->admin_q;
261 funq->cq_vector = 0;
262 rc = fun_request_irq(funq, devname: dev_name(dev: fdev->dev), handler: fun_admin_irq, data: funq);
263 if (rc)
264 goto free_sbq;
265
266 fun_set_cq_callback(funq, cb: fun_complete_admin_cmd, NULL);
267 fdev->adminq_cb = areq->event_cb;
268
269 writel(val: (funq->sq_depth - 1) << AQA_ASQS_SHIFT |
270 (funq->cq_depth - 1) << AQA_ACQS_SHIFT,
271 addr: fdev->bar + NVME_REG_AQA);
272
273 writeq(val: funq->sq_dma_addr, addr: fdev->bar + NVME_REG_ASQ);
274 writeq(val: funq->cq_dma_addr, addr: fdev->bar + NVME_REG_ACQ);
275
276 rc = fun_enable_ctrl(fdev, admin_cqesz_log2: areq->cqe_size_log2, admin_sqesz_log2: areq->sqe_size_log2);
277 if (rc)
278 goto free_irq;
279
280 if (areq->rq_depth) {
281 rc = fun_create_rq(funq);
282 if (rc)
283 goto disable_ctrl;
284
285 funq_rq_post(funq);
286 }
287
288 return 0;
289
290disable_ctrl:
291 fun_disable_ctrl(fdev);
292free_irq:
293 fun_free_irq(funq);
294free_sbq:
295 sbitmap_queue_free(sbq: &fdev->admin_sbq);
296free_cmd_ctx:
297 kvfree(addr: fdev->cmd_ctx);
298 fdev->cmd_ctx = NULL;
299free_q:
300 fun_free_queue(funq: fdev->admin_q);
301 fdev->admin_q = NULL;
302 return rc;
303}
304
305static void fun_disable_admin_queue(struct fun_dev *fdev)
306{
307 struct fun_queue *admq = fdev->admin_q;
308
309 if (!admq)
310 return;
311
312 fun_disable_ctrl(fdev);
313
314 fun_free_irq(funq: admq);
315 __fun_process_cq(funq: admq, max: 0);
316
317 sbitmap_queue_free(sbq: &fdev->admin_sbq);
318
319 kvfree(addr: fdev->cmd_ctx);
320 fdev->cmd_ctx = NULL;
321
322 fun_free_queue(funq: admq);
323 fdev->admin_q = NULL;
324}
325
326/* Return %true if the admin queue has stopped servicing commands as can be
327 * detected through registers. This isn't exhaustive and may provide false
328 * negatives.
329 */
330static bool fun_adminq_stopped(struct fun_dev *fdev)
331{
332 u32 csts = readl(addr: fdev->bar + NVME_REG_CSTS);
333
334 return (csts & (NVME_CSTS_CFS | NVME_CSTS_RDY)) != NVME_CSTS_RDY;
335}
336
337static int fun_wait_for_tag(struct fun_dev *fdev, int *cpup)
338{
339 struct sbitmap_queue *sbq = &fdev->admin_sbq;
340 struct sbq_wait_state *ws = &sbq->ws[0];
341 DEFINE_SBQ_WAIT(wait);
342 int tag;
343
344 for (;;) {
345 sbitmap_prepare_to_wait(sbq, ws, sbq_wait: &wait, TASK_UNINTERRUPTIBLE);
346 if (fdev->suppress_cmds) {
347 tag = -ESHUTDOWN;
348 break;
349 }
350 tag = sbitmap_queue_get(sbq, cpu: cpup);
351 if (tag >= 0)
352 break;
353 schedule();
354 }
355
356 sbitmap_finish_wait(sbq, ws, sbq_wait: &wait);
357 return tag;
358}
359
360/* Submit an asynchronous admin command. Caller is responsible for implementing
361 * any waiting or timeout. Upon command completion the callback @cb is called.
362 */
363int fun_submit_admin_cmd(struct fun_dev *fdev, struct fun_admin_req_common *cmd,
364 fun_admin_callback_t cb, void *cb_data, bool wait_ok)
365{
366 struct fun_queue *funq = fdev->admin_q;
367 unsigned int cmdsize = cmd->len8 * 8;
368 struct fun_cmd_ctx *cmd_ctx;
369 int tag, cpu, rc = 0;
370
371 if (WARN_ON(cmdsize > (1 << funq->sqe_size_log2)))
372 return -EMSGSIZE;
373
374 tag = sbitmap_queue_get(sbq: &fdev->admin_sbq, cpu: &cpu);
375 if (tag < 0) {
376 if (!wait_ok)
377 return -EAGAIN;
378 tag = fun_wait_for_tag(fdev, cpup: &cpu);
379 if (tag < 0)
380 return tag;
381 }
382
383 cmd->cid = cpu_to_be16(tag);
384
385 cmd_ctx = &fdev->cmd_ctx[tag];
386 cmd_ctx->cb = cb;
387 cmd_ctx->cb_data = cb_data;
388
389 spin_lock(lock: &funq->sq_lock);
390
391 if (unlikely(fdev->suppress_cmds)) {
392 rc = -ESHUTDOWN;
393 sbitmap_queue_clear(sbq: &fdev->admin_sbq, nr: tag, cpu);
394 } else {
395 cmd_ctx->cpu = cpu;
396 memcpy(fun_sqe_at(funq, funq->sq_tail), cmd, cmdsize);
397
398 dev_dbg(fdev->dev, "admin cmd @ %u: %8ph\n", funq->sq_tail,
399 cmd);
400
401 if (++funq->sq_tail == funq->sq_depth)
402 funq->sq_tail = 0;
403 writel(val: funq->sq_tail, addr: funq->sq_db);
404 }
405 spin_unlock(lock: &funq->sq_lock);
406 return rc;
407}
408
409/* Abandon a pending admin command by clearing the issuer's callback data.
410 * Failure indicates that the command either has already completed or its
411 * completion is racing with this call.
412 */
413static bool fun_abandon_admin_cmd(struct fun_dev *fd,
414 const struct fun_admin_req_common *cmd,
415 void *cb_data)
416{
417 u16 cid = be16_to_cpu(cmd->cid);
418 struct fun_cmd_ctx *cmd_ctx = &fd->cmd_ctx[cid];
419
420 return cmpxchg(&cmd_ctx->cb_data, cb_data, NULL) == cb_data;
421}
422
423/* Stop submission of new admin commands and wake up any processes waiting for
424 * tags. Already submitted commands are left to complete or time out.
425 */
426static void fun_admin_stop(struct fun_dev *fdev)
427{
428 spin_lock(lock: &fdev->admin_q->sq_lock);
429 fdev->suppress_cmds = true;
430 spin_unlock(lock: &fdev->admin_q->sq_lock);
431 sbitmap_queue_wake_all(sbq: &fdev->admin_sbq);
432}
433
434/* The callback for synchronous execution of admin commands. It copies the
435 * command response to the caller's buffer and signals completion.
436 */
437static void fun_admin_cmd_sync_cb(struct fun_dev *fd, void *rsp, void *cb_data)
438{
439 const struct fun_admin_rsp_common *rsp_common = rsp;
440 struct fun_sync_cmd_ctx *ctx = cb_data;
441
442 if (!ctx)
443 return; /* command issuer timed out and left */
444 if (ctx->rsp_buf) {
445 unsigned int rsp_len = rsp_common->len8 * 8;
446
447 if (unlikely(rsp_len > ctx->rsp_len)) {
448 dev_err(fd->dev,
449 "response for op %u is %uB > response buffer %uB\n",
450 rsp_common->op, rsp_len, ctx->rsp_len);
451 rsp_len = ctx->rsp_len;
452 }
453 memcpy(ctx->rsp_buf, rsp, rsp_len);
454 }
455 ctx->rsp_status = rsp_common->ret;
456 complete(&ctx->compl);
457}
458
459/* Submit a synchronous admin command. */
460int fun_submit_admin_sync_cmd(struct fun_dev *fdev,
461 struct fun_admin_req_common *cmd, void *rsp,
462 size_t rspsize, unsigned int timeout)
463{
464 struct fun_sync_cmd_ctx ctx = {
465 .compl = COMPLETION_INITIALIZER_ONSTACK(ctx.compl),
466 .rsp_buf = rsp,
467 .rsp_len = rspsize,
468 };
469 unsigned int cmdlen = cmd->len8 * 8;
470 unsigned long jiffies_left;
471 int ret;
472
473 ret = fun_submit_admin_cmd(fdev, cmd, cb: fun_admin_cmd_sync_cb, cb_data: &ctx,
474 wait_ok: true);
475 if (ret)
476 return ret;
477
478 if (!timeout)
479 timeout = FUN_ADMIN_CMD_TO_MS;
480
481 jiffies_left = wait_for_completion_timeout(x: &ctx.compl,
482 timeout: msecs_to_jiffies(m: timeout));
483 if (!jiffies_left) {
484 /* The command timed out. Attempt to cancel it so we can return.
485 * But if the command is in the process of completing we'll
486 * wait for it.
487 */
488 if (fun_abandon_admin_cmd(fd: fdev, cmd, cb_data: &ctx)) {
489 dev_err(fdev->dev, "admin command timed out: %*ph\n",
490 cmdlen, cmd);
491 fun_admin_stop(fdev);
492 /* see if the timeout was due to a queue failure */
493 if (fun_adminq_stopped(fdev))
494 dev_err(fdev->dev,
495 "device does not accept admin commands\n");
496
497 return -ETIMEDOUT;
498 }
499 wait_for_completion(&ctx.compl);
500 }
501
502 if (ctx.rsp_status) {
503 dev_err(fdev->dev, "admin command failed, err %d: %*ph\n",
504 ctx.rsp_status, cmdlen, cmd);
505 }
506
507 return -ctx.rsp_status;
508}
509EXPORT_SYMBOL_GPL(fun_submit_admin_sync_cmd);
510
511/* Return the number of device resources of the requested type. */
512int fun_get_res_count(struct fun_dev *fdev, enum fun_admin_op res)
513{
514 union {
515 struct fun_admin_res_count_req req;
516 struct fun_admin_res_count_rsp rsp;
517 } cmd;
518 int rc;
519
520 cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(cmd.req));
521 cmd.req.count = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_RES_COUNT,
522 0, 0);
523
524 rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common, &cmd.rsp,
525 sizeof(cmd), 0);
526 return rc ? rc : be32_to_cpu(cmd.rsp.count.data);
527}
528EXPORT_SYMBOL_GPL(fun_get_res_count);
529
530/* Request that the instance of resource @res with the given id be deleted. */
531int fun_res_destroy(struct fun_dev *fdev, enum fun_admin_op res,
532 unsigned int flags, u32 id)
533{
534 struct fun_admin_generic_destroy_req req = {
535 .common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(req)),
536 .destroy = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_DESTROY,
537 flags, id)
538 };
539
540 return fun_submit_admin_sync_cmd(fdev, &req.common, NULL, 0, 0);
541}
542EXPORT_SYMBOL_GPL(fun_res_destroy);
543
544/* Bind two entities of the given types and IDs. */
545int fun_bind(struct fun_dev *fdev, enum fun_admin_bind_type type0,
546 unsigned int id0, enum fun_admin_bind_type type1,
547 unsigned int id1)
548{
549 DEFINE_RAW_FLEX(struct fun_admin_bind_req, cmd, entry, 2);
550
551 cmd->common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_BIND,
552 __struct_size(cmd));
553 cmd->entry[0] = FUN_ADMIN_BIND_ENTRY_INIT(type0, id0);
554 cmd->entry[1] = FUN_ADMIN_BIND_ENTRY_INIT(type1, id1);
555
556 return fun_submit_admin_sync_cmd(fdev, &cmd->common, NULL, 0, 0);
557}
558EXPORT_SYMBOL_GPL(fun_bind);
559
560static int fun_get_dev_limits(struct fun_dev *fdev)
561{
562 struct pci_dev *pdev = to_pci_dev(fdev->dev);
563 unsigned int cq_count, sq_count, num_dbs;
564 int rc;
565
566 rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPCQ);
567 if (rc < 0)
568 return rc;
569 cq_count = rc;
570
571 rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPSQ);
572 if (rc < 0)
573 return rc;
574 sq_count = rc;
575
576 /* The admin queue consumes 1 CQ and at least 1 SQ. To be usable the
577 * device must provide additional queues.
578 */
579 if (cq_count < 2 || sq_count < 2 + !!fdev->admin_q->rq_depth)
580 return -EINVAL;
581
582 /* Calculate the max QID based on SQ/CQ/doorbell counts.
583 * SQ/CQ doorbells alternate.
584 */
585 num_dbs = (pci_resource_len(pdev, 0) - NVME_REG_DBS) >>
586 (2 + NVME_CAP_STRIDE(fdev->cap_reg));
587 fdev->max_qid = min3(cq_count, sq_count, num_dbs / 2) - 1;
588 fdev->kern_end_qid = fdev->max_qid + 1;
589 return 0;
590}
591
592/* Allocate all MSI-X vectors available on a function and at least @min_vecs. */
593static int fun_alloc_irqs(struct pci_dev *pdev, unsigned int min_vecs)
594{
595 int vecs, num_msix = pci_msix_vec_count(dev: pdev);
596
597 if (num_msix < 0)
598 return num_msix;
599 if (min_vecs > num_msix)
600 return -ERANGE;
601
602 vecs = pci_alloc_irq_vectors(dev: pdev, min_vecs, max_vecs: num_msix, PCI_IRQ_MSIX);
603 if (vecs > 0) {
604 dev_info(&pdev->dev,
605 "Allocated %d IRQ vectors of %d requested\n",
606 vecs, num_msix);
607 } else {
608 dev_err(&pdev->dev,
609 "Unable to allocate at least %u IRQ vectors\n",
610 min_vecs);
611 }
612 return vecs;
613}
614
615/* Allocate and initialize the IRQ manager state. */
616static int fun_alloc_irq_mgr(struct fun_dev *fdev)
617{
618 fdev->irq_map = bitmap_zalloc(nbits: fdev->num_irqs, GFP_KERNEL);
619 if (!fdev->irq_map)
620 return -ENOMEM;
621
622 spin_lock_init(&fdev->irqmgr_lock);
623 /* mark IRQ 0 allocated, it is used by the admin queue */
624 __set_bit(0, fdev->irq_map);
625 fdev->irqs_avail = fdev->num_irqs - 1;
626 return 0;
627}
628
629/* Reserve @nirqs of the currently available IRQs and return their indices. */
630int fun_reserve_irqs(struct fun_dev *fdev, unsigned int nirqs, u16 *irq_indices)
631{
632 unsigned int b, n = 0;
633 int err = -ENOSPC;
634
635 if (!nirqs)
636 return 0;
637
638 spin_lock(lock: &fdev->irqmgr_lock);
639 if (nirqs > fdev->irqs_avail)
640 goto unlock;
641
642 for_each_clear_bit(b, fdev->irq_map, fdev->num_irqs) {
643 __set_bit(b, fdev->irq_map);
644 irq_indices[n++] = b;
645 if (n >= nirqs)
646 break;
647 }
648
649 WARN_ON(n < nirqs);
650 fdev->irqs_avail -= n;
651 err = n;
652unlock:
653 spin_unlock(lock: &fdev->irqmgr_lock);
654 return err;
655}
656EXPORT_SYMBOL(fun_reserve_irqs);
657
658/* Release @nirqs previously allocated IRQS with the supplied indices. */
659void fun_release_irqs(struct fun_dev *fdev, unsigned int nirqs,
660 u16 *irq_indices)
661{
662 unsigned int i;
663
664 spin_lock(lock: &fdev->irqmgr_lock);
665 for (i = 0; i < nirqs; i++)
666 __clear_bit(irq_indices[i], fdev->irq_map);
667 fdev->irqs_avail += nirqs;
668 spin_unlock(lock: &fdev->irqmgr_lock);
669}
670EXPORT_SYMBOL(fun_release_irqs);
671
672static void fun_serv_handler(struct work_struct *work)
673{
674 struct fun_dev *fd = container_of(work, struct fun_dev, service_task);
675
676 if (test_bit(FUN_SERV_DISABLED, &fd->service_flags))
677 return;
678 if (fd->serv_cb)
679 fd->serv_cb(fd);
680}
681
682void fun_serv_stop(struct fun_dev *fd)
683{
684 set_bit(nr: FUN_SERV_DISABLED, addr: &fd->service_flags);
685 cancel_work_sync(work: &fd->service_task);
686}
687EXPORT_SYMBOL_GPL(fun_serv_stop);
688
689void fun_serv_restart(struct fun_dev *fd)
690{
691 clear_bit(nr: FUN_SERV_DISABLED, addr: &fd->service_flags);
692 if (fd->service_flags)
693 schedule_work(work: &fd->service_task);
694}
695EXPORT_SYMBOL_GPL(fun_serv_restart);
696
697void fun_serv_sched(struct fun_dev *fd)
698{
699 if (!test_bit(FUN_SERV_DISABLED, &fd->service_flags))
700 schedule_work(work: &fd->service_task);
701}
702EXPORT_SYMBOL_GPL(fun_serv_sched);
703
704/* Check and try to get the device into a proper state for initialization,
705 * i.e., CSTS.RDY = CC.EN = 0.
706 */
707static int sanitize_dev(struct fun_dev *fdev)
708{
709 int rc;
710
711 fdev->cap_reg = readq(addr: fdev->bar + NVME_REG_CAP);
712 fdev->cc_reg = readl(addr: fdev->bar + NVME_REG_CC);
713
714 /* First get RDY to agree with the current EN. Give RDY the opportunity
715 * to complete a potential recent EN change.
716 */
717 rc = fun_wait_ready(fdev, enabled: fdev->cc_reg & NVME_CC_ENABLE);
718 if (rc)
719 return rc;
720
721 /* Next, reset the device if EN is currently 1. */
722 if (fdev->cc_reg & NVME_CC_ENABLE)
723 rc = fun_disable_ctrl(fdev);
724
725 return rc;
726}
727
728/* Undo the device initialization of fun_dev_enable(). */
729void fun_dev_disable(struct fun_dev *fdev)
730{
731 struct pci_dev *pdev = to_pci_dev(fdev->dev);
732
733 pci_set_drvdata(pdev, NULL);
734
735 if (fdev->fw_handle != FUN_HCI_ID_INVALID) {
736 fun_res_destroy(fdev, FUN_ADMIN_OP_SWUPGRADE, 0,
737 fdev->fw_handle);
738 fdev->fw_handle = FUN_HCI_ID_INVALID;
739 }
740
741 fun_disable_admin_queue(fdev);
742
743 bitmap_free(bitmap: fdev->irq_map);
744 pci_free_irq_vectors(dev: pdev);
745
746 pci_disable_device(dev: pdev);
747
748 fun_unmap_bars(fdev);
749}
750EXPORT_SYMBOL(fun_dev_disable);
751
752/* Perform basic initialization of a device, including
753 * - PCI config space setup and BAR0 mapping
754 * - interrupt management initialization
755 * - 1 admin queue setup
756 * - determination of some device limits, such as number of queues.
757 */
758int fun_dev_enable(struct fun_dev *fdev, struct pci_dev *pdev,
759 const struct fun_dev_params *areq, const char *name)
760{
761 int rc;
762
763 fdev->dev = &pdev->dev;
764 rc = fun_map_bars(fdev, name);
765 if (rc)
766 return rc;
767
768 rc = fun_set_dma_masks(dev: fdev->dev);
769 if (rc)
770 goto unmap;
771
772 rc = pci_enable_device_mem(dev: pdev);
773 if (rc) {
774 dev_err(&pdev->dev, "Couldn't enable device, err %d\n", rc);
775 goto unmap;
776 }
777
778 rc = sanitize_dev(fdev);
779 if (rc)
780 goto disable_dev;
781
782 fdev->fw_handle = FUN_HCI_ID_INVALID;
783 fdev->q_depth = NVME_CAP_MQES(fdev->cap_reg) + 1;
784 fdev->db_stride = 1 << NVME_CAP_STRIDE(fdev->cap_reg);
785 fdev->dbs = fdev->bar + NVME_REG_DBS;
786
787 INIT_WORK(&fdev->service_task, fun_serv_handler);
788 fdev->service_flags = FUN_SERV_DISABLED;
789 fdev->serv_cb = areq->serv_cb;
790
791 rc = fun_alloc_irqs(pdev, min_vecs: areq->min_msix + 1); /* +1 for admin CQ */
792 if (rc < 0)
793 goto disable_dev;
794 fdev->num_irqs = rc;
795
796 rc = fun_alloc_irq_mgr(fdev);
797 if (rc)
798 goto free_irqs;
799
800 pci_set_master(dev: pdev);
801 rc = fun_enable_admin_queue(fdev, areq);
802 if (rc)
803 goto free_irq_mgr;
804
805 rc = fun_get_dev_limits(fdev);
806 if (rc < 0)
807 goto disable_admin;
808
809 pci_save_state(dev: pdev);
810 pci_set_drvdata(pdev, data: fdev);
811 pcie_print_link_status(dev: pdev);
812 dev_dbg(fdev->dev, "q_depth %u, db_stride %u, max qid %d kern_end_qid %d\n",
813 fdev->q_depth, fdev->db_stride, fdev->max_qid,
814 fdev->kern_end_qid);
815 return 0;
816
817disable_admin:
818 fun_disable_admin_queue(fdev);
819free_irq_mgr:
820 bitmap_free(bitmap: fdev->irq_map);
821free_irqs:
822 pci_free_irq_vectors(dev: pdev);
823disable_dev:
824 pci_disable_device(dev: pdev);
825unmap:
826 fun_unmap_bars(fdev);
827 return rc;
828}
829EXPORT_SYMBOL(fun_dev_enable);
830
831MODULE_AUTHOR("Dimitris Michailidis <dmichail@fungible.com>");
832MODULE_DESCRIPTION("Core services driver for Fungible devices");
833MODULE_LICENSE("Dual BSD/GPL");
834

source code of linux/drivers/net/ethernet/fungible/funcore/fun_dev.c