1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2020 Intel Corporation. */
3
4#include <linux/io-64-nonatomic-lo-hi.h>
5#include <linux/firmware.h>
6#include <linux/device.h>
7#include <linux/slab.h>
8#include <linux/idr.h>
9#include <linux/pci.h>
10#include <cxlmem.h>
11#include "trace.h"
12#include "core.h"
13
14static DECLARE_RWSEM(cxl_memdev_rwsem);
15
16/*
17 * An entire PCI topology full of devices should be enough for any
18 * config
19 */
20#define CXL_MEM_MAX_DEVS 65536
21
22static int cxl_mem_major;
23static DEFINE_IDA(cxl_memdev_ida);
24
25static void cxl_memdev_release(struct device *dev)
26{
27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
28
29 ida_free(&cxl_memdev_ida, id: cxlmd->id);
30 devm_cxl_memdev_edac_release(cxlmd);
31 kfree(objp: cxlmd);
32}
33
34static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,
35 kgid_t *gid)
36{
37 return kasprintf(GFP_KERNEL, fmt: "cxl/%s", dev_name(dev));
38}
39
40static ssize_t firmware_version_show(struct device *dev,
41 struct device_attribute *attr, char *buf)
42{
43 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
44 struct cxl_dev_state *cxlds = cxlmd->cxlds;
45 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
46
47 if (!mds)
48 return sysfs_emit(buf, fmt: "\n");
49 return sysfs_emit(buf, fmt: "%.16s\n", mds->firmware_version);
50}
51static DEVICE_ATTR_RO(firmware_version);
52
53static ssize_t payload_max_show(struct device *dev,
54 struct device_attribute *attr, char *buf)
55{
56 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
57 struct cxl_dev_state *cxlds = cxlmd->cxlds;
58 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
59
60 if (!mds)
61 return sysfs_emit(buf, fmt: "\n");
62 return sysfs_emit(buf, fmt: "%zu\n", cxlds->cxl_mbox.payload_size);
63}
64static DEVICE_ATTR_RO(payload_max);
65
66static ssize_t label_storage_size_show(struct device *dev,
67 struct device_attribute *attr, char *buf)
68{
69 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
70 struct cxl_dev_state *cxlds = cxlmd->cxlds;
71 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
72
73 if (!mds)
74 return sysfs_emit(buf, fmt: "\n");
75 return sysfs_emit(buf, fmt: "%zu\n", mds->lsa_size);
76}
77static DEVICE_ATTR_RO(label_storage_size);
78
79static resource_size_t cxl_ram_size(struct cxl_dev_state *cxlds)
80{
81 /* Static RAM is only expected at partition 0. */
82 if (cxlds->part[0].mode != CXL_PARTMODE_RAM)
83 return 0;
84 return resource_size(res: &cxlds->part[0].res);
85}
86
87static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr,
88 char *buf)
89{
90 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
91 struct cxl_dev_state *cxlds = cxlmd->cxlds;
92 unsigned long long len = cxl_ram_size(cxlds);
93
94 return sysfs_emit(buf, fmt: "%#llx\n", len);
95}
96
97static struct device_attribute dev_attr_ram_size =
98 __ATTR(size, 0444, ram_size_show, NULL);
99
100static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,
101 char *buf)
102{
103 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
104 struct cxl_dev_state *cxlds = cxlmd->cxlds;
105 unsigned long long len = cxl_pmem_size(cxlds);
106
107 return sysfs_emit(buf, fmt: "%#llx\n", len);
108}
109
110static struct device_attribute dev_attr_pmem_size =
111 __ATTR(size, 0444, pmem_size_show, NULL);
112
113static ssize_t serial_show(struct device *dev, struct device_attribute *attr,
114 char *buf)
115{
116 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
117 struct cxl_dev_state *cxlds = cxlmd->cxlds;
118
119 return sysfs_emit(buf, fmt: "%#llx\n", cxlds->serial);
120}
121static DEVICE_ATTR_RO(serial);
122
123static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,
124 char *buf)
125{
126 return sysfs_emit(buf, fmt: "%d\n", dev_to_node(dev));
127}
128static DEVICE_ATTR_RO(numa_node);
129
130static ssize_t security_state_show(struct device *dev,
131 struct device_attribute *attr,
132 char *buf)
133{
134 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
135 struct cxl_dev_state *cxlds = cxlmd->cxlds;
136 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
137 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
138 unsigned long state = mds->security.state;
139 int rc = 0;
140
141 /* sync with latest submission state */
142 mutex_lock(&cxl_mbox->mbox_mutex);
143 if (mds->security.sanitize_active)
144 rc = sysfs_emit(buf, fmt: "sanitize\n");
145 mutex_unlock(lock: &cxl_mbox->mbox_mutex);
146 if (rc)
147 return rc;
148
149 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET))
150 return sysfs_emit(buf, fmt: "disabled\n");
151 if (state & CXL_PMEM_SEC_STATE_FROZEN ||
152 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT ||
153 state & CXL_PMEM_SEC_STATE_USER_PLIMIT)
154 return sysfs_emit(buf, fmt: "frozen\n");
155 if (state & CXL_PMEM_SEC_STATE_LOCKED)
156 return sysfs_emit(buf, fmt: "locked\n");
157
158 return sysfs_emit(buf, fmt: "unlocked\n");
159}
160static struct device_attribute dev_attr_security_state =
161 __ATTR(state, 0444, security_state_show, NULL);
162
163static ssize_t security_sanitize_store(struct device *dev,
164 struct device_attribute *attr,
165 const char *buf, size_t len)
166{
167 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
168 bool sanitize;
169 ssize_t rc;
170
171 if (kstrtobool(s: buf, res: &sanitize) || !sanitize)
172 return -EINVAL;
173
174 rc = cxl_mem_sanitize(cxlmd, cmd: CXL_MBOX_OP_SANITIZE);
175 if (rc)
176 return rc;
177
178 return len;
179}
180static struct device_attribute dev_attr_security_sanitize =
181 __ATTR(sanitize, 0200, NULL, security_sanitize_store);
182
183static ssize_t security_erase_store(struct device *dev,
184 struct device_attribute *attr,
185 const char *buf, size_t len)
186{
187 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
188 ssize_t rc;
189 bool erase;
190
191 if (kstrtobool(s: buf, res: &erase) || !erase)
192 return -EINVAL;
193
194 rc = cxl_mem_sanitize(cxlmd, cmd: CXL_MBOX_OP_SECURE_ERASE);
195 if (rc)
196 return rc;
197
198 return len;
199}
200static struct device_attribute dev_attr_security_erase =
201 __ATTR(erase, 0200, NULL, security_erase_store);
202
203bool cxl_memdev_has_poison_cmd(struct cxl_memdev *cxlmd,
204 enum poison_cmd_enabled_bits cmd)
205{
206 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds: cxlmd->cxlds);
207
208 return test_bit(cmd, mds->poison.enabled_cmds);
209}
210
211static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd)
212{
213 struct cxl_dev_state *cxlds = cxlmd->cxlds;
214 u64 offset, length;
215 int rc = 0;
216
217 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */
218 for (int i = 0; i < cxlds->nr_partitions; i++) {
219 const struct resource *res = &cxlds->part[i].res;
220
221 offset = res->start;
222 length = resource_size(res);
223 rc = cxl_mem_get_poison(cxlmd, offset, len: length, NULL);
224 /*
225 * Invalid Physical Address is not an error for
226 * volatile addresses. Device support is optional.
227 */
228 if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM)
229 rc = 0;
230 }
231 return rc;
232}
233
234int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
235{
236 struct cxl_port *port;
237 int rc;
238
239 port = cxlmd->endpoint;
240 if (!port || !is_cxl_endpoint(port))
241 return -EINVAL;
242
243 ACQUIRE(rwsem_read_intr, region_rwsem)(T: &cxl_rwsem.region);
244 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
245 return rc;
246
247 ACQUIRE(rwsem_read_intr, dpa_rwsem)(T: &cxl_rwsem.dpa);
248 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
249 return rc;
250
251 if (cxl_num_decoders_committed(port) == 0) {
252 /* No regions mapped to this memdev */
253 rc = cxl_get_poison_by_memdev(cxlmd);
254 } else {
255 /* Regions mapped, collect poison by endpoint */
256 rc = cxl_get_poison_by_endpoint(port);
257 }
258
259 return rc;
260}
261EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, "CXL");
262
263static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)
264{
265 struct cxl_dev_state *cxlds = cxlmd->cxlds;
266
267 if (!IS_ENABLED(CONFIG_DEBUG_FS))
268 return 0;
269
270 if (!resource_size(res: &cxlds->dpa_res)) {
271 dev_dbg(cxlds->dev, "device has no dpa resource\n");
272 return -EINVAL;
273 }
274 if (!cxl_resource_contains_addr(res: &cxlds->dpa_res, addr: dpa)) {
275 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n",
276 dpa, &cxlds->dpa_res);
277 return -EINVAL;
278 }
279 if (!IS_ALIGNED(dpa, 64)) {
280 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa);
281 return -EINVAL;
282 }
283
284 return 0;
285}
286
287int cxl_inject_poison_locked(struct cxl_memdev *cxlmd, u64 dpa)
288{
289 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
290 struct cxl_mbox_inject_poison inject;
291 struct cxl_poison_record record;
292 struct cxl_mbox_cmd mbox_cmd;
293 struct cxl_region *cxlr;
294 int rc;
295
296 if (!IS_ENABLED(CONFIG_DEBUG_FS))
297 return 0;
298
299 lockdep_assert_held(&cxl_rwsem.dpa);
300 lockdep_assert_held(&cxl_rwsem.region);
301
302 rc = cxl_validate_poison_dpa(cxlmd, dpa);
303 if (rc)
304 return rc;
305
306 inject.address = cpu_to_le64(dpa);
307 mbox_cmd = (struct cxl_mbox_cmd) {
308 .opcode = CXL_MBOX_OP_INJECT_POISON,
309 .size_in = sizeof(inject),
310 .payload_in = &inject,
311 };
312 rc = cxl_internal_send_cmd(cxl_mbox, cmd: &mbox_cmd);
313 if (rc)
314 return rc;
315
316 cxlr = cxl_dpa_to_region(cxlmd, dpa);
317 if (cxlr)
318 dev_warn_once(cxl_mbox->host,
319 "poison inject dpa:%#llx region: %s\n", dpa,
320 dev_name(&cxlr->dev));
321
322 record = (struct cxl_poison_record) {
323 .address = cpu_to_le64(dpa),
324 .length = cpu_to_le32(1),
325 };
326 trace_cxl_poison(cxlmd, cxlr, record: &record, flags: 0, overflow_ts: 0, trace_type: CXL_POISON_TRACE_INJECT);
327
328 return 0;
329}
330
331int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
332{
333 int rc;
334
335 ACQUIRE(rwsem_read_intr, region_rwsem)(T: &cxl_rwsem.region);
336 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
337 return rc;
338
339 ACQUIRE(rwsem_read_intr, dpa_rwsem)(T: &cxl_rwsem.dpa);
340 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
341 return rc;
342
343 return cxl_inject_poison_locked(cxlmd, dpa);
344}
345EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL");
346
347int cxl_clear_poison_locked(struct cxl_memdev *cxlmd, u64 dpa)
348{
349 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
350 struct cxl_mbox_clear_poison clear;
351 struct cxl_poison_record record;
352 struct cxl_mbox_cmd mbox_cmd;
353 struct cxl_region *cxlr;
354 int rc;
355
356 if (!IS_ENABLED(CONFIG_DEBUG_FS))
357 return 0;
358
359 lockdep_assert_held(&cxl_rwsem.dpa);
360 lockdep_assert_held(&cxl_rwsem.region);
361
362 rc = cxl_validate_poison_dpa(cxlmd, dpa);
363 if (rc)
364 return rc;
365
366 /*
367 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command
368 * is defined to accept 64 bytes of write-data, along with the
369 * address to clear. This driver uses zeroes as write-data.
370 */
371 clear = (struct cxl_mbox_clear_poison) {
372 .address = cpu_to_le64(dpa)
373 };
374
375 mbox_cmd = (struct cxl_mbox_cmd) {
376 .opcode = CXL_MBOX_OP_CLEAR_POISON,
377 .size_in = sizeof(clear),
378 .payload_in = &clear,
379 };
380
381 rc = cxl_internal_send_cmd(cxl_mbox, cmd: &mbox_cmd);
382 if (rc)
383 return rc;
384
385 cxlr = cxl_dpa_to_region(cxlmd, dpa);
386 if (cxlr)
387 dev_warn_once(cxl_mbox->host,
388 "poison clear dpa:%#llx region: %s\n", dpa,
389 dev_name(&cxlr->dev));
390
391 record = (struct cxl_poison_record) {
392 .address = cpu_to_le64(dpa),
393 .length = cpu_to_le32(1),
394 };
395 trace_cxl_poison(cxlmd, cxlr, record: &record, flags: 0, overflow_ts: 0, trace_type: CXL_POISON_TRACE_CLEAR);
396
397 return 0;
398}
399
400int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
401{
402 int rc;
403
404 ACQUIRE(rwsem_read_intr, region_rwsem)(T: &cxl_rwsem.region);
405 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
406 return rc;
407
408 ACQUIRE(rwsem_read_intr, dpa_rwsem)(T: &cxl_rwsem.dpa);
409 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
410 return rc;
411
412 return cxl_clear_poison_locked(cxlmd, dpa);
413}
414EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL");
415
416static struct attribute *cxl_memdev_attributes[] = {
417 &dev_attr_serial.attr,
418 &dev_attr_firmware_version.attr,
419 &dev_attr_payload_max.attr,
420 &dev_attr_label_storage_size.attr,
421 &dev_attr_numa_node.attr,
422 NULL,
423};
424
425static struct cxl_dpa_perf *to_pmem_perf(struct cxl_dev_state *cxlds)
426{
427 for (int i = 0; i < cxlds->nr_partitions; i++)
428 if (cxlds->part[i].mode == CXL_PARTMODE_PMEM)
429 return &cxlds->part[i].perf;
430 return NULL;
431}
432
433static ssize_t pmem_qos_class_show(struct device *dev,
434 struct device_attribute *attr, char *buf)
435{
436 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
437 struct cxl_dev_state *cxlds = cxlmd->cxlds;
438
439 return sysfs_emit(buf, fmt: "%d\n", to_pmem_perf(cxlds)->qos_class);
440}
441
442static struct device_attribute dev_attr_pmem_qos_class =
443 __ATTR(qos_class, 0444, pmem_qos_class_show, NULL);
444
445static struct attribute *cxl_memdev_pmem_attributes[] = {
446 &dev_attr_pmem_size.attr,
447 &dev_attr_pmem_qos_class.attr,
448 NULL,
449};
450
451static struct cxl_dpa_perf *to_ram_perf(struct cxl_dev_state *cxlds)
452{
453 if (cxlds->part[0].mode != CXL_PARTMODE_RAM)
454 return NULL;
455 return &cxlds->part[0].perf;
456}
457
458static ssize_t ram_qos_class_show(struct device *dev,
459 struct device_attribute *attr, char *buf)
460{
461 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
462 struct cxl_dev_state *cxlds = cxlmd->cxlds;
463
464 return sysfs_emit(buf, fmt: "%d\n", to_ram_perf(cxlds)->qos_class);
465}
466
467static struct device_attribute dev_attr_ram_qos_class =
468 __ATTR(qos_class, 0444, ram_qos_class_show, NULL);
469
470static struct attribute *cxl_memdev_ram_attributes[] = {
471 &dev_attr_ram_size.attr,
472 &dev_attr_ram_qos_class.attr,
473 NULL,
474};
475
476static struct attribute *cxl_memdev_security_attributes[] = {
477 &dev_attr_security_state.attr,
478 &dev_attr_security_sanitize.attr,
479 &dev_attr_security_erase.attr,
480 NULL,
481};
482
483static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,
484 int n)
485{
486 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr)
487 return 0;
488 return a->mode;
489}
490
491static struct attribute_group cxl_memdev_attribute_group = {
492 .attrs = cxl_memdev_attributes,
493 .is_visible = cxl_memdev_visible,
494};
495
496static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)
497{
498 struct device *dev = kobj_to_dev(kobj);
499 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
500 struct cxl_dpa_perf *perf = to_ram_perf(cxlds: cxlmd->cxlds);
501
502 if (a == &dev_attr_ram_qos_class.attr &&
503 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID))
504 return 0;
505
506 return a->mode;
507}
508
509static struct attribute_group cxl_memdev_ram_attribute_group = {
510 .name = "ram",
511 .attrs = cxl_memdev_ram_attributes,
512 .is_visible = cxl_ram_visible,
513};
514
515static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n)
516{
517 struct device *dev = kobj_to_dev(kobj);
518 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
519 struct cxl_dpa_perf *perf = to_pmem_perf(cxlds: cxlmd->cxlds);
520
521 if (a == &dev_attr_pmem_qos_class.attr &&
522 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID))
523 return 0;
524
525 return a->mode;
526}
527
528static struct attribute_group cxl_memdev_pmem_attribute_group = {
529 .name = "pmem",
530 .attrs = cxl_memdev_pmem_attributes,
531 .is_visible = cxl_pmem_visible,
532};
533
534static umode_t cxl_memdev_security_visible(struct kobject *kobj,
535 struct attribute *a, int n)
536{
537 struct device *dev = kobj_to_dev(kobj);
538 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
539 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds: cxlmd->cxlds);
540
541 if (a == &dev_attr_security_sanitize.attr &&
542 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
543 return 0;
544
545 if (a == &dev_attr_security_erase.attr &&
546 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds))
547 return 0;
548
549 return a->mode;
550}
551
552static struct attribute_group cxl_memdev_security_attribute_group = {
553 .name = "security",
554 .attrs = cxl_memdev_security_attributes,
555 .is_visible = cxl_memdev_security_visible,
556};
557
558static const struct attribute_group *cxl_memdev_attribute_groups[] = {
559 &cxl_memdev_attribute_group,
560 &cxl_memdev_ram_attribute_group,
561 &cxl_memdev_pmem_attribute_group,
562 &cxl_memdev_security_attribute_group,
563 NULL,
564};
565
566void cxl_memdev_update_perf(struct cxl_memdev *cxlmd)
567{
568 sysfs_update_group(kobj: &cxlmd->dev.kobj, grp: &cxl_memdev_ram_attribute_group);
569 sysfs_update_group(kobj: &cxlmd->dev.kobj, grp: &cxl_memdev_pmem_attribute_group);
570}
571EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, "CXL");
572
573static const struct device_type cxl_memdev_type = {
574 .name = "cxl_memdev",
575 .release = cxl_memdev_release,
576 .devnode = cxl_memdev_devnode,
577 .groups = cxl_memdev_attribute_groups,
578};
579
580bool is_cxl_memdev(const struct device *dev)
581{
582 return dev->type == &cxl_memdev_type;
583}
584EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL");
585
586/**
587 * set_exclusive_cxl_commands() - atomically disable user cxl commands
588 * @mds: The device state to operate on
589 * @cmds: bitmap of commands to mark exclusive
590 *
591 * Grab the cxl_memdev_rwsem in write mode to flush in-flight
592 * invocations of the ioctl path and then disable future execution of
593 * commands with the command ids set in @cmds.
594 */
595void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
596 unsigned long *cmds)
597{
598 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
599
600 guard(rwsem_write)(T: &cxl_memdev_rwsem);
601 bitmap_or(dst: cxl_mbox->exclusive_cmds, src1: cxl_mbox->exclusive_cmds,
602 src2: cmds, nbits: CXL_MEM_COMMAND_ID_MAX);
603}
604EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, "CXL");
605
606/**
607 * clear_exclusive_cxl_commands() - atomically enable user cxl commands
608 * @mds: The device state to modify
609 * @cmds: bitmap of commands to mark available for userspace
610 */
611void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
612 unsigned long *cmds)
613{
614 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
615
616 guard(rwsem_write)(T: &cxl_memdev_rwsem);
617 bitmap_andnot(dst: cxl_mbox->exclusive_cmds, src1: cxl_mbox->exclusive_cmds,
618 src2: cmds, nbits: CXL_MEM_COMMAND_ID_MAX);
619}
620EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, "CXL");
621
622static void cxl_memdev_shutdown(struct device *dev)
623{
624 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
625
626 guard(rwsem_write)(T: &cxl_memdev_rwsem);
627 cxlmd->cxlds = NULL;
628}
629
630static void cxl_memdev_unregister(void *_cxlmd)
631{
632 struct cxl_memdev *cxlmd = _cxlmd;
633 struct device *dev = &cxlmd->dev;
634
635 cdev_device_del(cdev: &cxlmd->cdev, dev);
636 cxl_memdev_shutdown(dev);
637 put_device(dev);
638}
639
640static void detach_memdev(struct work_struct *work)
641{
642 struct cxl_memdev *cxlmd;
643
644 cxlmd = container_of(work, typeof(*cxlmd), detach_work);
645 device_release_driver(dev: &cxlmd->dev);
646 put_device(dev: &cxlmd->dev);
647}
648
649static struct lock_class_key cxl_memdev_key;
650
651static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
652 const struct file_operations *fops)
653{
654 struct cxl_memdev *cxlmd;
655 struct device *dev;
656 struct cdev *cdev;
657 int rc;
658
659 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL);
660 if (!cxlmd)
661 return ERR_PTR(error: -ENOMEM);
662
663 rc = ida_alloc_max(ida: &cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL);
664 if (rc < 0)
665 goto err;
666 cxlmd->id = rc;
667 cxlmd->depth = -1;
668
669 dev = &cxlmd->dev;
670 device_initialize(dev);
671 lockdep_set_class(&dev->mutex, &cxl_memdev_key);
672 dev->parent = cxlds->dev;
673 dev->bus = &cxl_bus_type;
674 dev->devt = MKDEV(cxl_mem_major, cxlmd->id);
675 dev->type = &cxl_memdev_type;
676 device_set_pm_not_required(dev);
677 INIT_WORK(&cxlmd->detach_work, detach_memdev);
678
679 cdev = &cxlmd->cdev;
680 cdev_init(cdev, fops);
681 return cxlmd;
682
683err:
684 kfree(objp: cxlmd);
685 return ERR_PTR(error: rc);
686}
687
688static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
689 unsigned long arg)
690{
691 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds: cxlmd->cxlds);
692 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
693
694 switch (cmd) {
695 case CXL_MEM_QUERY_COMMANDS:
696 return cxl_query_cmd(cxl_mbox, q: (void __user *)arg);
697 case CXL_MEM_SEND_COMMAND:
698 return cxl_send_cmd(cxl_mbox, s: (void __user *)arg);
699 default:
700 return -ENOTTY;
701 }
702}
703
704static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
705 unsigned long arg)
706{
707 struct cxl_memdev *cxlmd = file->private_data;
708 struct cxl_dev_state *cxlds;
709
710 guard(rwsem_read)(T: &cxl_memdev_rwsem);
711 cxlds = cxlmd->cxlds;
712 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM)
713 return __cxl_memdev_ioctl(cxlmd, cmd, arg);
714
715 return -ENXIO;
716}
717
718static int cxl_memdev_open(struct inode *inode, struct file *file)
719{
720 struct cxl_memdev *cxlmd =
721 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
722
723 get_device(dev: &cxlmd->dev);
724 file->private_data = cxlmd;
725
726 return 0;
727}
728
729static int cxl_memdev_release_file(struct inode *inode, struct file *file)
730{
731 struct cxl_memdev *cxlmd =
732 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
733
734 put_device(dev: &cxlmd->dev);
735
736 return 0;
737}
738
739/**
740 * cxl_mem_get_fw_info - Get Firmware info
741 * @mds: The device data for the operation
742 *
743 * Retrieve firmware info for the device specified.
744 *
745 * Return: 0 if no error: or the result of the mailbox command.
746 *
747 * See CXL-3.0 8.2.9.3.1 Get FW Info
748 */
749static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
750{
751 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
752 struct cxl_mbox_get_fw_info info;
753 struct cxl_mbox_cmd mbox_cmd;
754 int rc;
755
756 mbox_cmd = (struct cxl_mbox_cmd) {
757 .opcode = CXL_MBOX_OP_GET_FW_INFO,
758 .size_out = sizeof(info),
759 .payload_out = &info,
760 };
761
762 rc = cxl_internal_send_cmd(cxl_mbox, cmd: &mbox_cmd);
763 if (rc < 0)
764 return rc;
765
766 mds->fw.num_slots = info.num_slots;
767 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK,
768 info.slot_info);
769
770 return 0;
771}
772
773/**
774 * cxl_mem_activate_fw - Activate Firmware
775 * @mds: The device data for the operation
776 * @slot: slot number to activate
777 *
778 * Activate firmware in a given slot for the device specified.
779 *
780 * Return: 0 if no error: or the result of the mailbox command.
781 *
782 * See CXL-3.0 8.2.9.3.3 Activate FW
783 */
784static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
785{
786 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
787 struct cxl_mbox_activate_fw activate;
788 struct cxl_mbox_cmd mbox_cmd;
789
790 if (slot == 0 || slot > mds->fw.num_slots)
791 return -EINVAL;
792
793 mbox_cmd = (struct cxl_mbox_cmd) {
794 .opcode = CXL_MBOX_OP_ACTIVATE_FW,
795 .size_in = sizeof(activate),
796 .payload_in = &activate,
797 };
798
799 /* Only offline activation supported for now */
800 activate.action = CXL_FW_ACTIVATE_OFFLINE;
801 activate.slot = slot;
802
803 return cxl_internal_send_cmd(cxl_mbox, cmd: &mbox_cmd);
804}
805
806/**
807 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer
808 * @mds: The device data for the operation
809 *
810 * Abort an in-progress firmware transfer for the device specified.
811 *
812 * Return: 0 if no error: or the result of the mailbox command.
813 *
814 * See CXL-3.0 8.2.9.3.2 Transfer FW
815 */
816static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)
817{
818 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
819 struct cxl_mbox_transfer_fw *transfer;
820 struct cxl_mbox_cmd mbox_cmd;
821 int rc;
822
823 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL);
824 if (!transfer)
825 return -ENOMEM;
826
827 /* Set a 1s poll interval and a total wait time of 30s */
828 mbox_cmd = (struct cxl_mbox_cmd) {
829 .opcode = CXL_MBOX_OP_TRANSFER_FW,
830 .size_in = sizeof(*transfer),
831 .payload_in = transfer,
832 .poll_interval_ms = 1000,
833 .poll_count = 30,
834 };
835
836 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;
837
838 rc = cxl_internal_send_cmd(cxl_mbox, cmd: &mbox_cmd);
839 kfree(objp: transfer);
840 return rc;
841}
842
843static void cxl_fw_cleanup(struct fw_upload *fwl)
844{
845 struct cxl_memdev_state *mds = fwl->dd_handle;
846
847 mds->fw.next_slot = 0;
848}
849
850static int cxl_fw_do_cancel(struct fw_upload *fwl)
851{
852 struct cxl_memdev_state *mds = fwl->dd_handle;
853 struct cxl_dev_state *cxlds = &mds->cxlds;
854 struct cxl_memdev *cxlmd = cxlds->cxlmd;
855 int rc;
856
857 rc = cxl_mem_abort_fw_xfer(mds);
858 if (rc < 0)
859 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc);
860
861 return FW_UPLOAD_ERR_CANCELED;
862}
863
864static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
865 u32 size)
866{
867 struct cxl_memdev_state *mds = fwl->dd_handle;
868 struct cxl_mbox_transfer_fw *transfer;
869 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
870
871 if (!size)
872 return FW_UPLOAD_ERR_INVALID_SIZE;
873
874 mds->fw.oneshot = struct_size(transfer, data, size) <
875 cxl_mbox->payload_size;
876
877 if (cxl_mem_get_fw_info(mds))
878 return FW_UPLOAD_ERR_HW_ERROR;
879
880 /*
881 * So far no state has been changed, hence no other cleanup is
882 * necessary. Simply return the cancelled status.
883 */
884 if (test_and_clear_bit(CXL_FW_CANCEL, addr: mds->fw.state))
885 return FW_UPLOAD_ERR_CANCELED;
886
887 return FW_UPLOAD_ERR_NONE;
888}
889
890static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
891 u32 offset, u32 size, u32 *written)
892{
893 struct cxl_memdev_state *mds = fwl->dd_handle;
894 struct cxl_dev_state *cxlds = &mds->cxlds;
895 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
896 struct cxl_memdev *cxlmd = cxlds->cxlmd;
897 struct cxl_mbox_transfer_fw *transfer;
898 struct cxl_mbox_cmd mbox_cmd;
899 u32 cur_size, remaining;
900 size_t size_in;
901 int rc;
902
903 *written = 0;
904
905 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */
906 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) {
907 dev_err(&cxlmd->dev,
908 "misaligned offset for FW transfer slice (%u)\n",
909 offset);
910 return FW_UPLOAD_ERR_RW_ERROR;
911 }
912
913 /*
914 * Pick transfer size based on mds->payload_size @size must bw 128-byte
915 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and
916 * sizeof(*transfer) is 128. These constraints imply that @cur_size
917 * will always be 128b aligned.
918 */
919 cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer));
920
921 remaining = size - cur_size;
922 size_in = struct_size(transfer, data, cur_size);
923
924 if (test_and_clear_bit(CXL_FW_CANCEL, addr: mds->fw.state))
925 return cxl_fw_do_cancel(fwl);
926
927 /*
928 * Slot numbers are 1-indexed
929 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1')
930 * Check for rollover using modulo, and 1-index it by adding 1
931 */
932 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1;
933
934 /* Do the transfer via mailbox cmd */
935 transfer = kzalloc(size_in, GFP_KERNEL);
936 if (!transfer)
937 return FW_UPLOAD_ERR_RW_ERROR;
938
939 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT);
940 memcpy(transfer->data, data + offset, cur_size);
941 if (mds->fw.oneshot) {
942 transfer->action = CXL_FW_TRANSFER_ACTION_FULL;
943 transfer->slot = mds->fw.next_slot;
944 } else {
945 if (offset == 0) {
946 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE;
947 } else if (remaining == 0) {
948 transfer->action = CXL_FW_TRANSFER_ACTION_END;
949 transfer->slot = mds->fw.next_slot;
950 } else {
951 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE;
952 }
953 }
954
955 mbox_cmd = (struct cxl_mbox_cmd) {
956 .opcode = CXL_MBOX_OP_TRANSFER_FW,
957 .size_in = size_in,
958 .payload_in = transfer,
959 .poll_interval_ms = 1000,
960 .poll_count = 30,
961 };
962
963 rc = cxl_internal_send_cmd(cxl_mbox, cmd: &mbox_cmd);
964 if (rc < 0) {
965 rc = FW_UPLOAD_ERR_RW_ERROR;
966 goto out_free;
967 }
968
969 *written = cur_size;
970
971 /* Activate FW if oneshot or if the last slice was written */
972 if (mds->fw.oneshot || remaining == 0) {
973 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n",
974 mds->fw.next_slot);
975 rc = cxl_mem_activate_fw(mds, slot: mds->fw.next_slot);
976 if (rc < 0) {
977 dev_err(&cxlmd->dev, "Error activating firmware: %d\n",
978 rc);
979 rc = FW_UPLOAD_ERR_HW_ERROR;
980 goto out_free;
981 }
982 }
983
984 rc = FW_UPLOAD_ERR_NONE;
985
986out_free:
987 kfree(objp: transfer);
988 return rc;
989}
990
991static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl)
992{
993 struct cxl_memdev_state *mds = fwl->dd_handle;
994
995 /*
996 * cxl_internal_send_cmd() handles background operations synchronously.
997 * No need to wait for completions here - any errors would've been
998 * reported and handled during the ->write() call(s).
999 * Just check if a cancel request was received, and return success.
1000 */
1001 if (test_and_clear_bit(CXL_FW_CANCEL, addr: mds->fw.state))
1002 return cxl_fw_do_cancel(fwl);
1003
1004 return FW_UPLOAD_ERR_NONE;
1005}
1006
1007static void cxl_fw_cancel(struct fw_upload *fwl)
1008{
1009 struct cxl_memdev_state *mds = fwl->dd_handle;
1010
1011 set_bit(CXL_FW_CANCEL, addr: mds->fw.state);
1012}
1013
1014static const struct fw_upload_ops cxl_memdev_fw_ops = {
1015 .prepare = cxl_fw_prepare,
1016 .write = cxl_fw_write,
1017 .poll_complete = cxl_fw_poll_complete,
1018 .cancel = cxl_fw_cancel,
1019 .cleanup = cxl_fw_cleanup,
1020};
1021
1022static void cxl_remove_fw_upload(void *fwl)
1023{
1024 firmware_upload_unregister(fw_upload: fwl);
1025}
1026
1027int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds)
1028{
1029 struct cxl_dev_state *cxlds = &mds->cxlds;
1030 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
1031 struct device *dev = &cxlds->cxlmd->dev;
1032 struct fw_upload *fwl;
1033
1034 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, cxl_mbox->enabled_cmds))
1035 return 0;
1036
1037 fwl = firmware_upload_register(THIS_MODULE, parent: dev, name: dev_name(dev),
1038 ops: &cxl_memdev_fw_ops, dd_handle: mds);
1039 if (IS_ERR(ptr: fwl))
1040 return PTR_ERR(ptr: fwl);
1041 return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl);
1042}
1043EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, "CXL");
1044
1045static const struct file_operations cxl_memdev_fops = {
1046 .owner = THIS_MODULE,
1047 .unlocked_ioctl = cxl_memdev_ioctl,
1048 .open = cxl_memdev_open,
1049 .release = cxl_memdev_release_file,
1050 .compat_ioctl = compat_ptr_ioctl,
1051 .llseek = noop_llseek,
1052};
1053
1054struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
1055 struct cxl_dev_state *cxlds)
1056{
1057 struct cxl_memdev *cxlmd;
1058 struct device *dev;
1059 struct cdev *cdev;
1060 int rc;
1061
1062 cxlmd = cxl_memdev_alloc(cxlds, fops: &cxl_memdev_fops);
1063 if (IS_ERR(ptr: cxlmd))
1064 return cxlmd;
1065
1066 dev = &cxlmd->dev;
1067 rc = dev_set_name(dev, name: "mem%d", cxlmd->id);
1068 if (rc)
1069 goto err;
1070
1071 /*
1072 * Activate ioctl operations, no cxl_memdev_rwsem manipulation
1073 * needed as this is ordered with cdev_add() publishing the device.
1074 */
1075 cxlmd->cxlds = cxlds;
1076 cxlds->cxlmd = cxlmd;
1077
1078 cdev = &cxlmd->cdev;
1079 rc = cdev_device_add(cdev, dev);
1080 if (rc)
1081 goto err;
1082
1083 rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd);
1084 if (rc)
1085 return ERR_PTR(error: rc);
1086 return cxlmd;
1087
1088err:
1089 /*
1090 * The cdev was briefly live, shutdown any ioctl operations that
1091 * saw that state.
1092 */
1093 cxl_memdev_shutdown(dev);
1094 put_device(dev);
1095 return ERR_PTR(error: rc);
1096}
1097EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, "CXL");
1098
1099static void sanitize_teardown_notifier(void *data)
1100{
1101 struct cxl_memdev_state *mds = data;
1102 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1103 struct kernfs_node *state;
1104
1105 /*
1106 * Prevent new irq triggered invocations of the workqueue and
1107 * flush inflight invocations.
1108 */
1109 mutex_lock(&cxl_mbox->mbox_mutex);
1110 state = mds->security.sanitize_node;
1111 mds->security.sanitize_node = NULL;
1112 mutex_unlock(lock: &cxl_mbox->mbox_mutex);
1113
1114 cancel_delayed_work_sync(dwork: &mds->security.poll_dwork);
1115 sysfs_put(kn: state);
1116}
1117
1118int devm_cxl_sanitize_setup_notifier(struct device *host,
1119 struct cxl_memdev *cxlmd)
1120{
1121 struct cxl_dev_state *cxlds = cxlmd->cxlds;
1122 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
1123 struct kernfs_node *sec;
1124
1125 if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
1126 return 0;
1127
1128 /*
1129 * Note, the expectation is that @cxlmd would have failed to be
1130 * created if these sysfs_get_dirent calls fail.
1131 */
1132 sec = sysfs_get_dirent(parent: cxlmd->dev.kobj.sd, name: "security");
1133 if (!sec)
1134 return -ENOENT;
1135 mds->security.sanitize_node = sysfs_get_dirent(parent: sec, name: "state");
1136 sysfs_put(kn: sec);
1137 if (!mds->security.sanitize_node)
1138 return -ENOENT;
1139
1140 return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds);
1141}
1142EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, "CXL");
1143
1144__init int cxl_memdev_init(void)
1145{
1146 dev_t devt;
1147 int rc;
1148
1149 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl");
1150 if (rc)
1151 return rc;
1152
1153 cxl_mem_major = MAJOR(devt);
1154
1155 return 0;
1156}
1157
1158void cxl_memdev_exit(void)
1159{
1160 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS);
1161}
1162

source code of linux/drivers/cxl/core/memdev.c