1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Support for Realtek hardware ECC engine in RTL93xx SoCs
4 */
5
6#include <linux/bitfield.h>
7#include <linux/dma-mapping.h>
8#include <linux/mtd/nand.h>
9#include <linux/mutex.h>
10#include <linux/platform_device.h>
11#include <linux/regmap.h>
12
13/*
14 * The Realtek ECC engine has two operation modes.
15 *
16 * - BCH6 : Generate 10 ECC bytes from 512 data bytes plus 6 free bytes
17 * - BCH12 : Generate 20 ECC bytes from 512 data bytes plus 6 free bytes
18 *
19 * It can run for arbitrary NAND flash chips with different block and OOB sizes. Currently there
20 * are only two known devices in the wild that have NAND flash and make use of this ECC engine
21 * (Linksys LGS328C & LGS352C). To keep compatibility with vendor firmware, new modes can only
22 * be added when new data layouts have been analyzed. For now allow BCH6 on flash with 2048 byte
23 * blocks and 64 bytes oob.
24 *
25 * This driver aligns with kernel ECC naming conventions. Neverthless a short notice on the
26 * Realtek naming conventions for the different structures in the OOB area.
27 *
28 * - BBI : Bad block indicator. The first two bytes of OOB. Protected by ECC!
29 * - tag : 6 User/free bytes. First tag "contains" 2 bytes BBI. Protected by ECC!
30 * - syndrome : ECC/parity bytes
31 *
32 * Altogether this gives currently the following block layout.
33 *
34 * +------+------+------+------+-----+------+------+------+------+-----+-----+-----+-----+
35 * | 512 | 512 | 512 | 512 | 2 | 4 | 6 | 6 | 6 | 10 | 10 | 10 | 10 |
36 * +------+------+------+------+-----+------+------+------+------+-----+-----+-----+-----+
37 * | data | data | data | data | BBI | free | free | free | free | ECC | ECC | ECC | ECC |
38 * +------+------+------+------+-----+------+------+------+------+-----+-----+-----+-----+
39 */
40
41#define RTL_ECC_ALLOWED_PAGE_SIZE 2048
42#define RTL_ECC_ALLOWED_OOB_SIZE 64
43#define RTL_ECC_ALLOWED_STRENGTH 6
44
45#define RTL_ECC_BLOCK_SIZE 512
46#define RTL_ECC_FREE_SIZE 6
47#define RTL_ECC_PARITY_SIZE_BCH6 10
48#define RTL_ECC_PARITY_SIZE_BCH12 20
49
50/*
51 * The engine is fed with two DMA regions. One for data (always 512 bytes) and one for free bytes
52 * and parity (either 16 bytes for BCH6 or 26 bytes for BCH12). Start and length of each must be
53 * aligned to a multiple of 4.
54 */
55
56#define RTL_ECC_DMA_FREE_PARITY_SIZE ALIGN(RTL_ECC_FREE_SIZE + RTL_ECC_PARITY_SIZE_BCH12, 4)
57#define RTL_ECC_DMA_SIZE (RTL_ECC_BLOCK_SIZE + RTL_ECC_DMA_FREE_PARITY_SIZE)
58
59#define RTL_ECC_CFG 0x00
60#define RTL_ECC_BCH6 0
61#define RTL_ECC_BCH12 BIT(28)
62#define RTL_ECC_DMA_PRECISE BIT(12)
63#define RTL_ECC_BURST_128 GENMASK(1, 0)
64#define RTL_ECC_DMA_TRIGGER 0x08
65#define RTL_ECC_OP_DECODE 0
66#define RTL_ECC_OP_ENCODE BIT(0)
67#define RTL_ECC_DMA_START 0x0c
68#define RTL_ECC_DMA_TAG 0x10
69#define RTL_ECC_STATUS 0x14
70#define RTL_ECC_CORR_COUNT GENMASK(19, 12)
71#define RTL_ECC_RESULT BIT(8)
72#define RTL_ECC_ALL_ONE BIT(4)
73#define RTL_ECC_OP_STATUS BIT(0)
74
75struct rtl_ecc_engine {
76 struct device *dev;
77 struct nand_ecc_engine engine;
78 struct mutex lock;
79 char *buf;
80 dma_addr_t buf_dma;
81 struct regmap *regmap;
82};
83
84struct rtl_ecc_ctx {
85 struct rtl_ecc_engine * rtlc;
86 struct nand_ecc_req_tweak_ctx req_ctx;
87 int steps;
88 int bch_mode;
89 int strength;
90 int parity_size;
91};
92
93static const struct regmap_config rtl_ecc_regmap_config = {
94 .reg_bits = 32,
95 .val_bits = 32,
96 .reg_stride = 4,
97};
98
99static inline void *nand_to_ctx(struct nand_device *nand)
100{
101 return nand->ecc.ctx.priv;
102}
103
104static inline struct rtl_ecc_engine *nand_to_rtlc(struct nand_device *nand)
105{
106 struct nand_ecc_engine *eng = nand->ecc.engine;
107
108 return container_of(eng, struct rtl_ecc_engine, engine);
109}
110
111static int rtl_ecc_ooblayout_ecc(struct mtd_info *mtd, int section,
112 struct mtd_oob_region *oobregion)
113{
114 struct nand_device *nand = mtd_to_nanddev(mtd);
115 struct rtl_ecc_ctx *ctx = nand_to_ctx(nand);
116
117 if (section < 0 || section >= ctx->steps)
118 return -ERANGE;
119
120 oobregion->offset = ctx->steps * RTL_ECC_FREE_SIZE + section * ctx->parity_size;
121 oobregion->length = ctx->parity_size;
122
123 return 0;
124}
125
126static int rtl_ecc_ooblayout_free(struct mtd_info *mtd, int section,
127 struct mtd_oob_region *oobregion)
128{
129 struct nand_device *nand = mtd_to_nanddev(mtd);
130 struct rtl_ecc_ctx *ctx = nand_to_ctx(nand);
131 int bbm;
132
133 if (section < 0 || section >= ctx->steps)
134 return -ERANGE;
135
136 /* reserve 2 BBM bytes in first block */
137 bbm = section ? 0 : 2;
138 oobregion->offset = section * RTL_ECC_FREE_SIZE + bbm;
139 oobregion->length = RTL_ECC_FREE_SIZE - bbm;
140
141 return 0;
142}
143
144static const struct mtd_ooblayout_ops rtl_ecc_ooblayout_ops = {
145 .ecc = rtl_ecc_ooblayout_ecc,
146 .free = rtl_ecc_ooblayout_free,
147};
148
149static void rtl_ecc_kick_engine(struct rtl_ecc_ctx *ctx, int operation)
150{
151 struct rtl_ecc_engine *rtlc = ctx->rtlc;
152
153 regmap_write(map: rtlc->regmap, RTL_ECC_CFG,
154 val: ctx->bch_mode | RTL_ECC_BURST_128 | RTL_ECC_DMA_PRECISE);
155
156 regmap_write(map: rtlc->regmap, RTL_ECC_DMA_START, val: rtlc->buf_dma);
157 regmap_write(map: rtlc->regmap, RTL_ECC_DMA_TAG, val: rtlc->buf_dma + RTL_ECC_BLOCK_SIZE);
158 regmap_write(map: rtlc->regmap, RTL_ECC_DMA_TRIGGER, val: operation);
159}
160
161static int rtl_ecc_wait_for_engine(struct rtl_ecc_ctx *ctx)
162{
163 struct rtl_ecc_engine *rtlc = ctx->rtlc;
164 int ret, status, bitflips;
165 bool all_one;
166
167 /*
168 * The ECC engine needs 6-8 us to encode/decode a BCH6 syndrome for 512 bytes of data
169 * and 6 free bytes. In case the NAND area has been erased and all data and oob is
170 * set to 0xff, decoding takes 30us (reason unknown). Although the engine can trigger
171 * interrupts when finished, use active polling for now. 12 us maximum wait time has
172 * proven to be a good tradeoff between performance and overhead.
173 */
174
175 ret = regmap_read_poll_timeout(rtlc->regmap, RTL_ECC_STATUS, status,
176 !(status & RTL_ECC_OP_STATUS), 12, 1000000);
177 if (ret)
178 return ret;
179
180 ret = FIELD_GET(RTL_ECC_RESULT, status);
181 all_one = FIELD_GET(RTL_ECC_ALL_ONE, status);
182 bitflips = FIELD_GET(RTL_ECC_CORR_COUNT, status);
183
184 /* For erased blocks (all bits one) error status can be ignored */
185 if (all_one)
186 ret = 0;
187
188 return ret ? -EBADMSG : bitflips;
189}
190
191static int rtl_ecc_run_engine(struct rtl_ecc_ctx *ctx, char *data, char *free,
192 char *parity, int operation)
193{
194 struct rtl_ecc_engine *rtlc = ctx->rtlc;
195 char *buf_parity = rtlc->buf + RTL_ECC_BLOCK_SIZE + RTL_ECC_FREE_SIZE;
196 char *buf_free = rtlc->buf + RTL_ECC_BLOCK_SIZE;
197 char *buf_data = rtlc->buf;
198 int ret;
199
200 mutex_lock(&rtlc->lock);
201
202 memcpy(buf_data, data, RTL_ECC_BLOCK_SIZE);
203 memcpy(buf_free, free, RTL_ECC_FREE_SIZE);
204 memcpy(buf_parity, parity, ctx->parity_size);
205
206 dma_sync_single_for_device(dev: rtlc->dev, addr: rtlc->buf_dma, RTL_ECC_DMA_SIZE, dir: DMA_TO_DEVICE);
207 rtl_ecc_kick_engine(ctx, operation);
208 ret = rtl_ecc_wait_for_engine(ctx);
209 dma_sync_single_for_cpu(dev: rtlc->dev, addr: rtlc->buf_dma, RTL_ECC_DMA_SIZE, dir: DMA_FROM_DEVICE);
210
211 if (ret >= 0) {
212 memcpy(data, buf_data, RTL_ECC_BLOCK_SIZE);
213 memcpy(free, buf_free, RTL_ECC_FREE_SIZE);
214 memcpy(parity, buf_parity, ctx->parity_size);
215 }
216
217 mutex_unlock(lock: &rtlc->lock);
218
219 return ret;
220}
221
222static int rtl_ecc_prepare_io_req(struct nand_device *nand, struct nand_page_io_req *req)
223{
224 struct rtl_ecc_engine *rtlc = nand_to_rtlc(nand);
225 struct rtl_ecc_ctx *ctx = nand_to_ctx(nand);
226 char *data, *free, *parity;
227 int ret = 0;
228
229 if (req->mode == MTD_OPS_RAW)
230 return 0;
231
232 nand_ecc_tweak_req(ctx: &ctx->req_ctx, req);
233
234 if (req->type == NAND_PAGE_READ)
235 return 0;
236
237 free = req->oobbuf.in;
238 data = req->databuf.in;
239 parity = req->oobbuf.in + ctx->steps * RTL_ECC_FREE_SIZE;
240
241 for (int i = 0; i < ctx->steps; i++) {
242 ret |= rtl_ecc_run_engine(ctx, data, free, parity, RTL_ECC_OP_ENCODE);
243
244 free += RTL_ECC_FREE_SIZE;
245 data += RTL_ECC_BLOCK_SIZE;
246 parity += ctx->parity_size;
247 }
248
249 if (unlikely(ret))
250 dev_dbg(rtlc->dev, "ECC calculation failed\n");
251
252 return ret ? -EBADMSG : 0;
253}
254
255static int rtl_ecc_finish_io_req(struct nand_device *nand, struct nand_page_io_req *req)
256{
257 struct rtl_ecc_engine *rtlc = nand_to_rtlc(nand);
258 struct rtl_ecc_ctx *ctx = nand_to_ctx(nand);
259 struct mtd_info *mtd = nanddev_to_mtd(nand);
260 char *data, *free, *parity;
261 bool failure = false;
262 int bitflips = 0;
263
264 if (req->mode == MTD_OPS_RAW)
265 return 0;
266
267 if (req->type == NAND_PAGE_WRITE) {
268 nand_ecc_restore_req(ctx: &ctx->req_ctx, req);
269 return 0;
270 }
271
272 free = req->oobbuf.in;
273 data = req->databuf.in;
274 parity = req->oobbuf.in + ctx->steps * RTL_ECC_FREE_SIZE;
275
276 for (int i = 0 ; i < ctx->steps; i++) {
277 int ret = rtl_ecc_run_engine(ctx, data, free, parity, RTL_ECC_OP_DECODE);
278
279 if (unlikely(ret < 0))
280 /* ECC totally fails for bitflips in erased blocks */
281 ret = nand_check_erased_ecc_chunk(data, RTL_ECC_BLOCK_SIZE,
282 ecc: parity, ecclen: ctx->parity_size,
283 extraoob: free, RTL_ECC_FREE_SIZE,
284 threshold: ctx->strength);
285 if (unlikely(ret < 0)) {
286 failure = true;
287 mtd->ecc_stats.failed++;
288 } else {
289 mtd->ecc_stats.corrected += ret;
290 bitflips = max_t(unsigned int, bitflips, ret);
291 }
292
293 free += RTL_ECC_FREE_SIZE;
294 data += RTL_ECC_BLOCK_SIZE;
295 parity += ctx->parity_size;
296 }
297
298 nand_ecc_restore_req(ctx: &ctx->req_ctx, req);
299
300 if (unlikely(failure))
301 dev_dbg(rtlc->dev, "ECC correction failed\n");
302 else if (unlikely(bitflips > 2))
303 dev_dbg(rtlc->dev, "%d bitflips detected\n", bitflips);
304
305 return failure ? -EBADMSG : bitflips;
306}
307
308static int rtl_ecc_check_support(struct nand_device *nand)
309{
310 struct mtd_info *mtd = nanddev_to_mtd(nand);
311 struct device *dev = nand->ecc.engine->dev;
312
313 if (mtd->oobsize != RTL_ECC_ALLOWED_OOB_SIZE ||
314 mtd->writesize != RTL_ECC_ALLOWED_PAGE_SIZE) {
315 dev_err(dev, "only flash geometry data=%d, oob=%d supported\n",
316 RTL_ECC_ALLOWED_PAGE_SIZE, RTL_ECC_ALLOWED_OOB_SIZE);
317 return -EINVAL;
318 }
319
320 if (nand->ecc.user_conf.algo != NAND_ECC_ALGO_BCH ||
321 nand->ecc.user_conf.strength != RTL_ECC_ALLOWED_STRENGTH ||
322 nand->ecc.user_conf.placement != NAND_ECC_PLACEMENT_OOB ||
323 nand->ecc.user_conf.step_size != RTL_ECC_BLOCK_SIZE) {
324 dev_err(dev, "only algo=bch, strength=%d, placement=oob, step=%d supported\n",
325 RTL_ECC_ALLOWED_STRENGTH, RTL_ECC_BLOCK_SIZE);
326 return -EINVAL;
327 }
328
329 return 0;
330}
331
332static int rtl_ecc_init_ctx(struct nand_device *nand)
333{
334 struct nand_ecc_props *conf = &nand->ecc.ctx.conf;
335 struct rtl_ecc_engine *rtlc = nand_to_rtlc(nand);
336 struct mtd_info *mtd = nanddev_to_mtd(nand);
337 int strength = nand->ecc.user_conf.strength;
338 struct device *dev = nand->ecc.engine->dev;
339 struct rtl_ecc_ctx *ctx;
340 int ret;
341
342 ret = rtl_ecc_check_support(nand);
343 if (ret)
344 return ret;
345
346 ctx = devm_kzalloc(dev, size: sizeof(*ctx), GFP_KERNEL);
347 if (!ctx)
348 return -ENOMEM;
349
350 nand->ecc.ctx.priv = ctx;
351 mtd_set_ooblayout(mtd, ooblayout: &rtl_ecc_ooblayout_ops);
352
353 conf->algo = NAND_ECC_ALGO_BCH;
354 conf->strength = strength;
355 conf->step_size = RTL_ECC_BLOCK_SIZE;
356 conf->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
357
358 ctx->rtlc = rtlc;
359 ctx->steps = mtd->writesize / RTL_ECC_BLOCK_SIZE;
360 ctx->strength = strength;
361 ctx->bch_mode = strength == 6 ? RTL_ECC_BCH6 : RTL_ECC_BCH12;
362 ctx->parity_size = strength == 6 ? RTL_ECC_PARITY_SIZE_BCH6 : RTL_ECC_PARITY_SIZE_BCH12;
363
364 ret = nand_ecc_init_req_tweaking(ctx: &ctx->req_ctx, nand);
365 if (ret)
366 return ret;
367
368 dev_dbg(dev, "using bch%d with geometry data=%dx%d, free=%dx6, parity=%dx%d",
369 conf->strength, ctx->steps, conf->step_size,
370 ctx->steps, ctx->steps, ctx->parity_size);
371
372 return 0;
373}
374
375static void rtl_ecc_cleanup_ctx(struct nand_device *nand)
376{
377 struct rtl_ecc_ctx *ctx = nand_to_ctx(nand);
378
379 if (ctx)
380 nand_ecc_cleanup_req_tweaking(ctx: &ctx->req_ctx);
381}
382
383static const struct nand_ecc_engine_ops rtl_ecc_engine_ops = {
384 .init_ctx = rtl_ecc_init_ctx,
385 .cleanup_ctx = rtl_ecc_cleanup_ctx,
386 .prepare_io_req = rtl_ecc_prepare_io_req,
387 .finish_io_req = rtl_ecc_finish_io_req,
388};
389
390static int rtl_ecc_probe(struct platform_device *pdev)
391{
392 struct device *dev = &pdev->dev;
393 struct rtl_ecc_engine *rtlc;
394 void __iomem *base;
395 int ret;
396
397 rtlc = devm_kzalloc(dev, size: sizeof(*rtlc), GFP_KERNEL);
398 if (!rtlc)
399 return -ENOMEM;
400
401 base = devm_platform_ioremap_resource(pdev, index: 0);
402 if (IS_ERR(ptr: base))
403 return PTR_ERR(ptr: base);
404
405 ret = devm_mutex_init(dev, &rtlc->lock);
406 if (ret)
407 return ret;
408
409 rtlc->regmap = devm_regmap_init_mmio(dev, base, &rtl_ecc_regmap_config);
410 if (IS_ERR(ptr: rtlc->regmap))
411 return PTR_ERR(ptr: rtlc->regmap);
412
413 /*
414 * Focus on simplicity and use a preallocated DMA buffer for data exchange with the
415 * engine. For now make it a noncoherent memory model as invalidating/flushing caches
416 * is faster than reading/writing uncached memory on the known architectures.
417 */
418
419 rtlc->buf = dma_alloc_noncoherent(dev, RTL_ECC_DMA_SIZE, dma_handle: &rtlc->buf_dma,
420 dir: DMA_BIDIRECTIONAL, GFP_KERNEL);
421 if (!rtlc->buf)
422 return -ENOMEM;
423
424 rtlc->dev = dev;
425 rtlc->engine.dev = dev;
426 rtlc->engine.ops = &rtl_ecc_engine_ops;
427 rtlc->engine.integration = NAND_ECC_ENGINE_INTEGRATION_EXTERNAL;
428
429 nand_ecc_register_on_host_hw_engine(engine: &rtlc->engine);
430
431 platform_set_drvdata(pdev, data: rtlc);
432
433 return 0;
434}
435
436static void rtl_ecc_remove(struct platform_device *pdev)
437{
438 struct rtl_ecc_engine *rtlc = platform_get_drvdata(pdev);
439
440 nand_ecc_unregister_on_host_hw_engine(engine: &rtlc->engine);
441 dma_free_noncoherent(dev: rtlc->dev, RTL_ECC_DMA_SIZE, vaddr: rtlc->buf, dma_handle: rtlc->buf_dma,
442 dir: DMA_BIDIRECTIONAL);
443}
444
445static const struct of_device_id rtl_ecc_of_ids[] = {
446 {
447 .compatible = "realtek,rtl9301-ecc",
448 },
449 { /* sentinel */ },
450};
451
452static struct platform_driver rtl_ecc_driver = {
453 .driver = {
454 .name = "rtl-nand-ecc-engine",
455 .of_match_table = rtl_ecc_of_ids,
456 },
457 .probe = rtl_ecc_probe,
458 .remove = rtl_ecc_remove,
459};
460module_platform_driver(rtl_ecc_driver);
461
462MODULE_LICENSE("GPL");
463MODULE_AUTHOR("Markus Stockhausen <markus.stockhausen@gmx.de>");
464MODULE_DESCRIPTION("Realtek NAND hardware ECC controller");
465

source code of linux/drivers/mtd/nand/ecc-realtek.c