1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright 2017 ATMEL
4 * Copyright 2017 Free Electrons
5 *
6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
7 *
8 * Derived from the atmel_nand.c driver which contained the following
9 * copyrights:
10 *
11 * Copyright 2003 Rick Bronson
12 *
13 * Derived from drivers/mtd/nand/autcpu12.c (removed in v3.8)
14 * Copyright 2001 Thomas Gleixner (gleixner@autronix.de)
15 *
16 * Derived from drivers/mtd/spia.c (removed in v3.8)
17 * Copyright 2000 Steven J. Hill (sjhill@cotw.com)
18 *
19 * Add Hardware ECC support for AT91SAM9260 / AT91SAM9263
20 * Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007
21 *
22 * Derived from Das U-Boot source code
23 * (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c)
24 * Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas
25 *
26 * Add Programmable Multibit ECC support for various AT91 SoC
27 * Copyright 2012 ATMEL, Hong Xu
28 *
29 * Add Nand Flash Controller support for SAMA5 SoC
30 * Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com)
31 *
32 * The PMECC is an hardware assisted BCH engine, which means part of the
33 * ECC algorithm is left to the software. The hardware/software repartition
34 * is explained in the "PMECC Controller Functional Description" chapter in
35 * Atmel datasheets, and some of the functions in this file are directly
36 * implementing the algorithms described in the "Software Implementation"
37 * sub-section.
38 *
39 * TODO: it seems that the software BCH implementation in lib/bch.c is already
40 * providing some of the logic we are implementing here. It would be smart
41 * to expose the needed lib/bch.c helpers/functions and re-use them here.
42 */
43 #include <linux/iopoll.h>
44 #include <linux/mtd/rawnand.h>
45 #include <dm/of_access.h>
46 #include <dm/ofnode.h>
47 #include <asm/io.h>
48 #include "pmecc.h"
49 #include <dm/uclass.h>
50 #include <dm/device_compat.h>
51 #include <dm/devres.h>
52 #include <linux/ioport.h>
53
54 /* Galois field dimension */
55 #define PMECC_GF_DIMENSION_13 13
56 #define PMECC_GF_DIMENSION_14 14
57
58 /* Primitive Polynomial used by PMECC */
59 #define PMECC_GF_13_PRIMITIVE_POLY 0x201b
60 #define PMECC_GF_14_PRIMITIVE_POLY 0x4443
61
62 #define PMECC_LOOKUP_TABLE_SIZE_512 0x2000
63 #define PMECC_LOOKUP_TABLE_SIZE_1024 0x4000
64
65 /* Time out value for reading PMECC status register */
66 #define PMECC_MAX_TIMEOUT_MS 100
67
68 /* PMECC Register Definitions */
69 #define ATMEL_PMECC_CFG 0x0
70 #define PMECC_CFG_BCH_STRENGTH(x) (x)
71 #define PMECC_CFG_BCH_STRENGTH_MASK GENMASK(2, 0)
72 #define PMECC_CFG_SECTOR512 (0 << 4)
73 #define PMECC_CFG_SECTOR1024 BIT(4)
74 #define PMECC_CFG_NSECTORS(x) ((fls(x) - 1) << 8)
75 #define PMECC_CFG_READ_OP (0 << 12)
76 #define PMECC_CFG_WRITE_OP BIT(12)
77 #define PMECC_CFG_SPARE_ENABLE BIT(16)
78 #define PMECC_CFG_AUTO_ENABLE BIT(20)
79
80 #define ATMEL_PMECC_SAREA 0x4
81 #define ATMEL_PMECC_SADDR 0x8
82 #define ATMEL_PMECC_EADDR 0xc
83
84 #define ATMEL_PMECC_CLK 0x10
85 #define PMECC_CLK_133MHZ (2 << 0)
86
87 #define ATMEL_PMECC_CTRL 0x14
88 #define PMECC_CTRL_RST BIT(0)
89 #define PMECC_CTRL_DATA BIT(1)
90 #define PMECC_CTRL_USER BIT(2)
91 #define PMECC_CTRL_ENABLE BIT(4)
92 #define PMECC_CTRL_DISABLE BIT(5)
93
94 #define ATMEL_PMECC_SR 0x18
95 #define PMECC_SR_BUSY BIT(0)
96 #define PMECC_SR_ENABLE BIT(4)
97
98 #define ATMEL_PMECC_IER 0x1c
99 #define ATMEL_PMECC_IDR 0x20
100 #define ATMEL_PMECC_IMR 0x24
101 #define ATMEL_PMECC_ISR 0x28
102 #define PMECC_ERROR_INT BIT(0)
103
104 #define ATMEL_PMECC_ECC(sector, n) \
105 ((((sector) + 1) * 0x40) + (n))
106
107 #define ATMEL_PMECC_REM(sector, n) \
108 ((((sector) + 1) * 0x40) + ((n) * 4) + 0x200)
109
110 /* PMERRLOC Register Definitions */
111 #define ATMEL_PMERRLOC_ELCFG 0x0
112 #define PMERRLOC_ELCFG_SECTOR_512 (0 << 0)
113 #define PMERRLOC_ELCFG_SECTOR_1024 BIT(0)
114 #define PMERRLOC_ELCFG_NUM_ERRORS(n) ((n) << 16)
115
116 #define ATMEL_PMERRLOC_ELPRIM 0x4
117 #define ATMEL_PMERRLOC_ELEN 0x8
118 #define ATMEL_PMERRLOC_ELDIS 0xc
119 #define PMERRLOC_DISABLE BIT(0)
120
121 #define ATMEL_PMERRLOC_ELSR 0x10
122 #define PMERRLOC_ELSR_BUSY BIT(0)
123
124 #define ATMEL_PMERRLOC_ELIER 0x14
125 #define ATMEL_PMERRLOC_ELIDR 0x18
126 #define ATMEL_PMERRLOC_ELIMR 0x1c
127 #define ATMEL_PMERRLOC_ELISR 0x20
128 #define PMERRLOC_ERR_NUM_MASK GENMASK(12, 8)
129 #define PMERRLOC_CALC_DONE BIT(0)
130
131 #define ATMEL_PMERRLOC_SIGMA(x) (((x) * 0x4) + 0x28)
132
133 #define ATMEL_PMERRLOC_EL(offs, x) (((x) * 0x4) + (offs))
134
135 struct atmel_pmecc_gf_tables {
136 u16 *alpha_to;
137 u16 *index_of;
138 };
139
140 struct atmel_pmecc_caps {
141 const int *strengths;
142 int nstrengths;
143 int el_offset;
144 bool correct_erased_chunks;
145 };
146
147 struct atmel_pmecc_user_conf_cache {
148 u32 cfg;
149 u32 sarea;
150 u32 saddr;
151 u32 eaddr;
152 };
153
154 struct atmel_pmecc_user {
155 struct atmel_pmecc_user_conf_cache cache;
156 struct atmel_pmecc *pmecc;
157 const struct atmel_pmecc_gf_tables *gf_tables;
158 int eccbytes;
159 s16 *partial_syn;
160 s16 *si;
161 s16 *lmu;
162 s16 *smu;
163 s32 *mu;
164 s32 *dmu;
165 s32 *delta;
166 u32 isr;
167 };
168
169 /* Serialize table access */
170 DEFINE_MUTEX(pmecc_gf_tables_lock);
171 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512;
172 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024;
173
deg(unsigned int poly)174 static inline int deg(unsigned int poly)
175 {
176 /* polynomial degree is the most-significant bit index */
177 return fls(poly) - 1;
178 }
179
atmel_pmecc_build_gf_tables(int mm,unsigned int poly,struct atmel_pmecc_gf_tables * gf_tables)180 static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly,
181 struct atmel_pmecc_gf_tables *gf_tables)
182 {
183 unsigned int i, x = 1;
184 const unsigned int k = BIT(deg(poly));
185 unsigned int nn = BIT(mm) - 1;
186
187 /* primitive polynomial must be of degree m */
188 if (k != (1u << mm))
189 return -EINVAL;
190
191 for (i = 0; i < nn; i++) {
192 gf_tables->alpha_to[i] = x;
193 gf_tables->index_of[x] = i;
194 if (i && (x == 1))
195 /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */
196 return -EINVAL;
197 x <<= 1;
198 if (x & k)
199 x ^= poly;
200 }
201 gf_tables->alpha_to[nn] = 1;
202 gf_tables->index_of[0] = 0;
203
204 return 0;
205 }
206
207 static const struct atmel_pmecc_gf_tables *
atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req * req)208 atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req)
209 {
210 struct atmel_pmecc_gf_tables *gf_tables;
211 unsigned int poly, degree, table_size;
212 int ret;
213
214 if (req->ecc.sectorsize == 512) {
215 degree = PMECC_GF_DIMENSION_13;
216 poly = PMECC_GF_13_PRIMITIVE_POLY;
217 table_size = PMECC_LOOKUP_TABLE_SIZE_512;
218 } else {
219 degree = PMECC_GF_DIMENSION_14;
220 poly = PMECC_GF_14_PRIMITIVE_POLY;
221 table_size = PMECC_LOOKUP_TABLE_SIZE_1024;
222 }
223
224 gf_tables = kzalloc(sizeof(*gf_tables) +
225 (2 * table_size * sizeof(u16)),
226 GFP_KERNEL);
227 if (!gf_tables)
228 return ERR_PTR(-ENOMEM);
229
230 gf_tables->alpha_to = (void *)(gf_tables + 1);
231 gf_tables->index_of = gf_tables->alpha_to + table_size;
232
233 ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables);
234 if (ret) {
235 kfree(gf_tables);
236 return ERR_PTR(ret);
237 }
238
239 return gf_tables;
240 }
241
242 static const struct atmel_pmecc_gf_tables *
atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req * req)243 atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req)
244 {
245 const struct atmel_pmecc_gf_tables **gf_tables, *ret;
246
247 mutex_lock(&pmecc_gf_tables_lock);
248 if (req->ecc.sectorsize == 512)
249 gf_tables = &pmecc_gf_tables_512;
250 else
251 gf_tables = &pmecc_gf_tables_1024;
252
253 ret = *gf_tables;
254
255 if (!ret) {
256 ret = atmel_pmecc_create_gf_tables(req);
257 if (!IS_ERR(ret))
258 *gf_tables = ret;
259 }
260 mutex_unlock(&pmecc_gf_tables_lock);
261
262 return ret;
263 }
264
atmel_pmecc_prepare_user_req(struct atmel_pmecc * pmecc,struct atmel_pmecc_user_req * req)265 static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc,
266 struct atmel_pmecc_user_req *req)
267 {
268 int i, max_eccbytes, eccbytes = 0, eccstrength = 0;
269
270 if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0)
271 return -EINVAL;
272
273 if (req->ecc.ooboffset >= 0 &&
274 req->ecc.ooboffset + req->ecc.bytes > req->oobsize)
275 return -EINVAL;
276
277 if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) {
278 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
279 return -EINVAL;
280
281 if (req->pagesize > 512)
282 req->ecc.sectorsize = 1024;
283 else
284 req->ecc.sectorsize = 512;
285 }
286
287 if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024)
288 return -EINVAL;
289
290 if (req->pagesize % req->ecc.sectorsize)
291 return -EINVAL;
292
293 req->ecc.nsectors = req->pagesize / req->ecc.sectorsize;
294
295 max_eccbytes = req->ecc.bytes;
296
297 for (i = 0; i < pmecc->caps->nstrengths; i++) {
298 int nbytes, strength = pmecc->caps->strengths[i];
299
300 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH &&
301 strength < req->ecc.strength)
302 continue;
303
304 nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize),
305 8);
306 nbytes *= req->ecc.nsectors;
307
308 if (nbytes > max_eccbytes)
309 break;
310
311 eccstrength = strength;
312 eccbytes = nbytes;
313
314 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
315 break;
316 }
317
318 if (!eccstrength)
319 return -EINVAL;
320
321 req->ecc.bytes = eccbytes;
322 req->ecc.strength = eccstrength;
323
324 if (req->ecc.ooboffset < 0)
325 req->ecc.ooboffset = req->oobsize - eccbytes;
326
327 return 0;
328 }
329
330 struct atmel_pmecc_user *
atmel_pmecc_create_user(struct atmel_pmecc * pmecc,struct atmel_pmecc_user_req * req)331 atmel_pmecc_create_user(struct atmel_pmecc *pmecc,
332 struct atmel_pmecc_user_req *req)
333 {
334 struct atmel_pmecc_user *user;
335 const struct atmel_pmecc_gf_tables *gf_tables;
336 int strength, size, ret;
337
338 ret = atmel_pmecc_prepare_user_req(pmecc, req);
339 if (ret)
340 return ERR_PTR(ret);
341
342 size = sizeof(*user);
343 size = ALIGN(size, sizeof(u16));
344 /* Reserve space for partial_syn, si and smu */
345 size += ((2 * req->ecc.strength) + 1) * sizeof(u16) *
346 (2 + req->ecc.strength + 2);
347 /* Reserve space for lmu. */
348 size += (req->ecc.strength + 1) * sizeof(u16);
349 /* Reserve space for mu, dmu and delta. */
350 size = ALIGN(size, sizeof(s32));
351 size += (req->ecc.strength + 1) * sizeof(s32) * 3;
352
353 user = kzalloc(size, GFP_KERNEL);
354 if (!user)
355 return ERR_PTR(-ENOMEM);
356
357 user->pmecc = pmecc;
358
359 user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16));
360 user->si = user->partial_syn + ((2 * req->ecc.strength) + 1);
361 user->lmu = user->si + ((2 * req->ecc.strength) + 1);
362 user->smu = user->lmu + (req->ecc.strength + 1);
363 user->mu = (s32 *)PTR_ALIGN(user->smu +
364 (((2 * req->ecc.strength) + 1) *
365 (req->ecc.strength + 2)),
366 sizeof(s32));
367 user->dmu = user->mu + req->ecc.strength + 1;
368 user->delta = user->dmu + req->ecc.strength + 1;
369
370 gf_tables = atmel_pmecc_get_gf_tables(req);
371 if (IS_ERR(gf_tables)) {
372 kfree(user);
373 return ERR_CAST(gf_tables);
374 }
375
376 user->gf_tables = gf_tables;
377
378 user->eccbytes = req->ecc.bytes / req->ecc.nsectors;
379
380 for (strength = 0; strength < pmecc->caps->nstrengths; strength++) {
381 if (pmecc->caps->strengths[strength] == req->ecc.strength)
382 break;
383 }
384
385 user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) |
386 PMECC_CFG_NSECTORS(req->ecc.nsectors);
387
388 if (req->ecc.sectorsize == 1024)
389 user->cache.cfg |= PMECC_CFG_SECTOR1024;
390
391 user->cache.sarea = req->oobsize - 1;
392 user->cache.saddr = req->ecc.ooboffset;
393 user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1;
394
395 return user;
396 }
397 EXPORT_SYMBOL_GPL(atmel_pmecc_create_user);
398
atmel_pmecc_destroy_user(struct atmel_pmecc_user * user)399 void atmel_pmecc_destroy_user(struct atmel_pmecc_user *user)
400 {
401 kfree(user);
402 }
403 EXPORT_SYMBOL_GPL(atmel_pmecc_destroy_user);
404
get_strength(struct atmel_pmecc_user * user)405 static int get_strength(struct atmel_pmecc_user *user)
406 {
407 const int *strengths = user->pmecc->caps->strengths;
408
409 return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK];
410 }
411
get_sectorsize(struct atmel_pmecc_user * user)412 static int get_sectorsize(struct atmel_pmecc_user *user)
413 {
414 return user->cache.cfg & PMECC_CFG_SECTOR1024 ? 1024 : 512;
415 }
416
atmel_pmecc_gen_syndrome(struct atmel_pmecc_user * user,int sector)417 static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector)
418 {
419 int strength = get_strength(user);
420 u32 value;
421 int i;
422
423 /* Fill odd syndromes */
424 for (i = 0; i < strength; i++) {
425 value = readl_relaxed(user->pmecc->regs.base +
426 ATMEL_PMECC_REM(sector, i / 2));
427 if (i & 1)
428 value >>= 16;
429
430 user->partial_syn[(2 * i) + 1] = value;
431 }
432 }
433
atmel_pmecc_substitute(struct atmel_pmecc_user * user)434 static void atmel_pmecc_substitute(struct atmel_pmecc_user *user)
435 {
436 int degree = get_sectorsize(user) == 512 ? 13 : 14;
437 int cw_len = BIT(degree) - 1;
438 int strength = get_strength(user);
439 s16 *alpha_to = (s16 *)user->gf_tables->alpha_to;
440 s16 *index_of = (s16 *)user->gf_tables->index_of;
441 s16 *partial_syn = user->partial_syn;
442 s16 *si;
443 int i, j;
444
445 /*
446 * si[] is a table that holds the current syndrome value,
447 * an element of that table belongs to the field
448 */
449 si = user->si;
450
451 memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1));
452
453 /* Computation 2t syndromes based on S(x) */
454 /* Odd syndromes */
455 for (i = 1; i < 2 * strength; i += 2) {
456 for (j = 0; j < degree; j++) {
457 if (partial_syn[i] & BIT(j))
458 si[i] = alpha_to[i * j] ^ si[i];
459 }
460 }
461 /* Even syndrome = (Odd syndrome) ** 2 */
462 for (i = 2, j = 1; j <= strength; i = ++j << 1) {
463 if (si[j] == 0) {
464 si[i] = 0;
465 } else {
466 s16 tmp;
467
468 tmp = index_of[si[j]];
469 tmp = (tmp * 2) % cw_len;
470 si[i] = alpha_to[tmp];
471 }
472 }
473 }
474
atmel_pmecc_get_sigma(struct atmel_pmecc_user * user)475 static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user)
476 {
477 s16 *lmu = user->lmu;
478 s16 *si = user->si;
479 s32 *mu = user->mu;
480 s32 *dmu = user->dmu;
481 s32 *delta = user->delta;
482 int degree = get_sectorsize(user) == 512 ? 13 : 14;
483 int cw_len = BIT(degree) - 1;
484 int strength = get_strength(user);
485 int num = 2 * strength + 1;
486 s16 *index_of = (s16 *)user->gf_tables->index_of;
487 s16 *alpha_to = (s16 *)user->gf_tables->alpha_to;
488 int i, j, k;
489 u32 dmu_0_count, tmp;
490 s16 *smu = user->smu;
491
492 /* index of largest delta */
493 int ro;
494 int largest;
495 int diff;
496
497 dmu_0_count = 0;
498
499 /* First Row */
500
501 /* Mu */
502 mu[0] = -1;
503
504 memset(smu, 0, sizeof(s16) * num);
505 smu[0] = 1;
506
507 /* discrepancy set to 1 */
508 dmu[0] = 1;
509 /* polynom order set to 0 */
510 lmu[0] = 0;
511 delta[0] = (mu[0] * 2 - lmu[0]) >> 1;
512
513 /* Second Row */
514
515 /* Mu */
516 mu[1] = 0;
517 /* Sigma(x) set to 1 */
518 memset(&smu[num], 0, sizeof(s16) * num);
519 smu[num] = 1;
520
521 /* discrepancy set to S1 */
522 dmu[1] = si[1];
523
524 /* polynom order set to 0 */
525 lmu[1] = 0;
526
527 delta[1] = (mu[1] * 2 - lmu[1]) >> 1;
528
529 /* Init the Sigma(x) last row */
530 memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num);
531
532 for (i = 1; i <= strength; i++) {
533 mu[i + 1] = i << 1;
534 /* Begin Computing Sigma (Mu+1) and L(mu) */
535 /* check if discrepancy is set to 0 */
536 if (dmu[i] == 0) {
537 dmu_0_count++;
538
539 tmp = ((strength - (lmu[i] >> 1) - 1) / 2);
540 if ((strength - (lmu[i] >> 1) - 1) & 0x1)
541 tmp += 2;
542 else
543 tmp += 1;
544
545 if (dmu_0_count == tmp) {
546 for (j = 0; j <= (lmu[i] >> 1) + 1; j++)
547 smu[(strength + 1) * num + j] =
548 smu[i * num + j];
549
550 lmu[strength + 1] = lmu[i];
551 return;
552 }
553
554 /* copy polynom */
555 for (j = 0; j <= lmu[i] >> 1; j++)
556 smu[(i + 1) * num + j] = smu[i * num + j];
557
558 /* copy previous polynom order to the next */
559 lmu[i + 1] = lmu[i];
560 } else {
561 ro = 0;
562 largest = -1;
563 /* find largest delta with dmu != 0 */
564 for (j = 0; j < i; j++) {
565 if ((dmu[j]) && (delta[j] > largest)) {
566 largest = delta[j];
567 ro = j;
568 }
569 }
570
571 /* compute difference */
572 diff = (mu[i] - mu[ro]);
573
574 /* Compute degree of the new smu polynomial */
575 if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff))
576 lmu[i + 1] = lmu[i];
577 else
578 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2;
579
580 /* Init smu[i+1] with 0 */
581 for (k = 0; k < num; k++)
582 smu[(i + 1) * num + k] = 0;
583
584 /* Compute smu[i+1] */
585 for (k = 0; k <= lmu[ro] >> 1; k++) {
586 s16 a, b, c;
587
588 if (!(smu[ro * num + k] && dmu[i]))
589 continue;
590
591 a = index_of[dmu[i]];
592 b = index_of[dmu[ro]];
593 c = index_of[smu[ro * num + k]];
594 tmp = a + (cw_len - b) + c;
595 a = alpha_to[tmp % cw_len];
596 smu[(i + 1) * num + (k + diff)] = a;
597 }
598
599 for (k = 0; k <= lmu[i] >> 1; k++)
600 smu[(i + 1) * num + k] ^= smu[i * num + k];
601 }
602
603 /* End Computing Sigma (Mu+1) and L(mu) */
604 /* In either case compute delta */
605 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1;
606
607 /* Do not compute discrepancy for the last iteration */
608 if (i >= strength)
609 continue;
610
611 for (k = 0; k <= (lmu[i + 1] >> 1); k++) {
612 tmp = 2 * (i - 1);
613 if (k == 0) {
614 dmu[i + 1] = si[tmp + 3];
615 } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) {
616 s16 a, b, c;
617
618 a = index_of[smu[(i + 1) * num + k]];
619 b = si[2 * (i - 1) + 3 - k];
620 c = index_of[b];
621 tmp = a + c;
622 tmp %= cw_len;
623 dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1];
624 }
625 }
626 }
627 }
628
atmel_pmecc_err_location(struct atmel_pmecc_user * user)629 static int atmel_pmecc_err_location(struct atmel_pmecc_user *user)
630 {
631 int sector_size = get_sectorsize(user);
632 int degree = sector_size == 512 ? 13 : 14;
633 struct atmel_pmecc *pmecc = user->pmecc;
634 int strength = get_strength(user);
635 int ret, roots_nbr, i, err_nbr = 0;
636 int num = (2 * strength) + 1;
637 s16 *smu = user->smu;
638 u32 val;
639
640 writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS);
641
642 for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) {
643 writel_relaxed(smu[(strength + 1) * num + i],
644 pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i));
645 err_nbr++;
646 }
647
648 val = (err_nbr - 1) << 16;
649 if (sector_size == 1024)
650 val |= 1;
651
652 writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG);
653 writel((sector_size * 8) + (degree * strength),
654 pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN);
655
656 ret = readl_relaxed_poll_timeout(pmecc->regs.errloc +
657 ATMEL_PMERRLOC_ELISR,
658 val, val & PMERRLOC_CALC_DONE,
659 PMECC_MAX_TIMEOUT_MS * 1000);
660 if (ret) {
661 dev_err(pmecc->dev,
662 "PMECC: Timeout to calculate error location.\n");
663 return ret;
664 }
665
666 roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8;
667 /* Number of roots == degree of smu hence <= cap */
668 if (roots_nbr == user->lmu[strength + 1] >> 1)
669 return err_nbr - 1;
670
671 /*
672 * Number of roots does not match the degree of smu
673 * unable to correct error.
674 */
675 return -EBADMSG;
676 }
677
atmel_pmecc_correct_sector(struct atmel_pmecc_user * user,int sector,void * data,void * ecc)678 int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector,
679 void *data, void *ecc)
680 {
681 struct atmel_pmecc *pmecc = user->pmecc;
682 int sectorsize = get_sectorsize(user);
683 int eccbytes = user->eccbytes;
684 int i, nerrors;
685
686 if (!(user->isr & BIT(sector)))
687 return 0;
688
689 atmel_pmecc_gen_syndrome(user, sector);
690 atmel_pmecc_substitute(user);
691 atmel_pmecc_get_sigma(user);
692
693 nerrors = atmel_pmecc_err_location(user);
694 if (nerrors < 0)
695 return nerrors;
696
697 for (i = 0; i < nerrors; i++) {
698 const char *area;
699 int byte, bit;
700 u32 errpos;
701 u8 *ptr;
702
703 errpos = readl_relaxed(pmecc->regs.errloc +
704 ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i));
705 errpos--;
706
707 byte = errpos / 8;
708 bit = errpos % 8;
709
710 if (byte < sectorsize) {
711 ptr = data + byte;
712 area = "data";
713 } else if (byte < sectorsize + eccbytes) {
714 ptr = ecc + byte - sectorsize;
715 area = "ECC";
716 } else {
717 dev_dbg(pmecc->dev,
718 "Invalid errpos value (%d, max is %d)\n",
719 errpos, (sectorsize + eccbytes) * 8);
720 return -EINVAL;
721 }
722
723 dev_dbg(pmecc->dev,
724 "Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n",
725 area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit)));
726
727 *ptr ^= BIT(bit);
728 }
729
730 return nerrors;
731 }
732 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector);
733
atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user * user)734 bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user)
735 {
736 return user->pmecc->caps->correct_erased_chunks;
737 }
738 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks);
739
atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user * user,int sector,void * ecc)740 void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user,
741 int sector, void *ecc)
742 {
743 struct atmel_pmecc *pmecc = user->pmecc;
744 u8 *ptr = ecc;
745 int i;
746
747 for (i = 0; i < user->eccbytes; i++)
748 ptr[i] = readb_relaxed(pmecc->regs.base +
749 ATMEL_PMECC_ECC(sector, i));
750 }
751 EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes);
752
atmel_pmecc_reset(struct atmel_pmecc * pmecc)753 void atmel_pmecc_reset(struct atmel_pmecc *pmecc)
754 {
755 writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL);
756 writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
757 }
758 EXPORT_SYMBOL_GPL(atmel_pmecc_reset);
759
atmel_pmecc_enable(struct atmel_pmecc_user * user,int op)760 int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op)
761 {
762 struct atmel_pmecc *pmecc = user->pmecc;
763 u32 cfg;
764
765 if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) {
766 dev_err(pmecc->dev, "Bad ECC operation!");
767 return -EINVAL;
768 }
769
770 mutex_lock(&user->pmecc->lock);
771
772 cfg = user->cache.cfg;
773 if (op == NAND_ECC_WRITE)
774 cfg |= PMECC_CFG_WRITE_OP;
775 else
776 cfg |= PMECC_CFG_AUTO_ENABLE;
777
778 writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG);
779 writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA);
780 writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR);
781 writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR);
782
783 writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
784 writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL);
785
786 return 0;
787 }
788 EXPORT_SYMBOL_GPL(atmel_pmecc_enable);
789
atmel_pmecc_disable(struct atmel_pmecc_user * user)790 void atmel_pmecc_disable(struct atmel_pmecc_user *user)
791 {
792 atmel_pmecc_reset(user->pmecc);
793 mutex_unlock(&user->pmecc->lock);
794 }
795 EXPORT_SYMBOL_GPL(atmel_pmecc_disable);
796
atmel_pmecc_wait_rdy(struct atmel_pmecc_user * user)797 int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user)
798 {
799 struct atmel_pmecc *pmecc = user->pmecc;
800 u32 status;
801 int ret;
802
803 ret = readl_relaxed_poll_timeout(pmecc->regs.base +
804 ATMEL_PMECC_SR,
805 status, !(status & PMECC_SR_BUSY),
806 PMECC_MAX_TIMEOUT_MS * 1000);
807 if (ret) {
808 dev_err(pmecc->dev,
809 "Timeout while waiting for PMECC ready.\n");
810 return ret;
811 }
812
813 user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR);
814
815 return 0;
816 }
817 EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy);
818
819 #define ATMEL_BASE_PMECC 0xffffe000
820 #define ATMEL_BASE_PMERRLOC 0xffffe600
821
822 static struct atmel_pmecc *
atmel_pmecc_create(struct udevice * dev,const struct atmel_pmecc_caps * caps,int pmecc_res_idx,int errloc_res_idx,int timing_res_idx)823 atmel_pmecc_create(struct udevice *dev,
824 const struct atmel_pmecc_caps *caps,
825 int pmecc_res_idx, int errloc_res_idx,
826 int timing_res_idx)
827 {
828 struct atmel_pmecc *pmecc;
829 struct resource res;
830
831 pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL);
832 if (!pmecc)
833 return ERR_PTR(-ENOMEM);
834
835 pmecc->caps = caps;
836 pmecc->dev = dev;
837 mutex_init(&pmecc->lock);
838
839 ofnode_read_resource(dev->node_, 0, &res);
840 pmecc->regs.base = (void *)res.start;
841 ofnode_read_resource(dev->node_, 1, &res);
842 pmecc->regs.errloc = (void *)res.start;
843
844 pmecc->regs.timing = 0;
845
846 /* Disable all interrupts before registering the PMECC handler. */
847 writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR);
848 atmel_pmecc_reset(pmecc);
849
850 return pmecc;
851 }
852
devm_atmel_pmecc_put(struct udevice * dev,void * res)853 static void devm_atmel_pmecc_put(struct udevice *dev, void *res)
854 {
855 }
856
atmel_pmecc_get_by_node(struct udevice * dev)857 static struct atmel_pmecc *atmel_pmecc_get_by_node(struct udevice *dev)
858 {
859 struct atmel_pmecc *pmecc, **ptr;
860 int ret;
861
862 pmecc = dev_get_plat(dev);
863 if (!pmecc) {
864 ret = -EPROBE_DEFER;
865 goto err_put_device;
866 }
867
868 ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL);
869 if (!ptr) {
870 ret = -ENOMEM;
871 goto err_put_device;
872 }
873
874 *ptr = pmecc;
875
876 devres_add(dev, ptr);
877
878 return pmecc;
879
880 err_put_device:
881 return ERR_PTR(ret);
882 }
883
884 static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 };
885
886 static struct atmel_pmecc_caps at91sam9g45_caps = {
887 .strengths = atmel_pmecc_strengths,
888 .nstrengths = 5,
889 .el_offset = 0x8c,
890 };
891
892 static struct atmel_pmecc_caps sama5d4_caps = {
893 .strengths = atmel_pmecc_strengths,
894 .nstrengths = 5,
895 .el_offset = 0x8c,
896 .correct_erased_chunks = true,
897 };
898
899 static struct atmel_pmecc_caps sama5d2_caps = {
900 .strengths = atmel_pmecc_strengths,
901 .nstrengths = 6,
902 .el_offset = 0xac,
903 .correct_erased_chunks = true,
904 };
905
devm_atmel_pmecc_get(struct udevice * userdev)906 struct atmel_pmecc *devm_atmel_pmecc_get(struct udevice *userdev)
907 {
908 struct atmel_pmecc *pmecc;
909 struct ofnode_phandle_args args;
910 struct udevice *pdev;
911 int ret;
912
913 if (!userdev)
914 return ERR_PTR(-EINVAL);
915
916 ret = ofnode_parse_phandle_with_args(userdev->node_,
917 "ecc-engine",
918 NULL, 0, 0, &args);
919 ret = uclass_get_device_by_ofnode(UCLASS_MTD, args.node, &pdev);
920 if (ret)
921 return NULL;
922
923 pmecc = atmel_pmecc_get_by_node(pdev);
924
925 /* TODO:
926 * Support old DT bindings
927 */
928
929 return pmecc;
930 }
931 EXPORT_SYMBOL(devm_atmel_pmecc_get);
932
933 static const struct udevice_id atmel_pmecc_match[] = {
934 { .compatible = "atmel,at91sam9g45-pmecc", (ulong)&at91sam9g45_caps },
935 { .compatible = "atmel,sama5d4-pmecc", (ulong)&sama5d4_caps },
936 { .compatible = "atmel,sama5d2-pmecc", (ulong)&sama5d2_caps },
937 { /* sentinel */ }
938 };
939
atmel_pmecc_probe(struct udevice * dev)940 static int atmel_pmecc_probe(struct udevice *dev)
941 {
942 const struct atmel_pmecc_caps *caps;
943 struct atmel_pmecc *pmecc;
944
945 caps = (struct atmel_pmecc_caps *)dev_get_driver_data(dev);
946 if (!caps) {
947 dev_err(dev, "Invalid caps\n");
948 return -EINVAL;
949 }
950
951 pmecc = atmel_pmecc_create(dev, caps, 0, 1, 2);
952 if (IS_ERR(pmecc))
953 return PTR_ERR(pmecc);
954
955 dev->plat_ = pmecc;
956
957 return 0;
958 }
959
960 U_BOOT_DRIVER(atmel_pmecc) = {
961 .name = "atmel-pmecc",
962 .id = UCLASS_MTD,
963 .of_match = atmel_pmecc_match,
964 .probe = atmel_pmecc_probe,
965 };
966