1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
4 * Synopsys DesignWare eDMA v0 core
5 *
6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
7 */
8
9 #include <linux/bitfield.h>
10
11 #include <linux/io-64-nonatomic-lo-hi.h>
12
13 #include "dw-edma-core.h"
14 #include "dw-edma-v0-core.h"
15 #include "dw-edma-v0-regs.h"
16 #include "dw-edma-v0-debugfs.h"
17
18 enum dw_edma_control {
19 DW_EDMA_V0_CB = BIT(0),
20 DW_EDMA_V0_TCB = BIT(1),
21 DW_EDMA_V0_LLP = BIT(2),
22 DW_EDMA_V0_LIE = BIT(3),
23 DW_EDMA_V0_RIE = BIT(4),
24 DW_EDMA_V0_CCS = BIT(8),
25 DW_EDMA_V0_LLE = BIT(9),
26 };
27
__dw_regs(struct dw_edma * dw)28 static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw)
29 {
30 return dw->chip->reg_base;
31 }
32
33 #define SET_32(dw, name, value) \
34 writel(value, &(__dw_regs(dw)->name))
35
36 #define GET_32(dw, name) \
37 readl(&(__dw_regs(dw)->name))
38
39 #define SET_RW_32(dw, dir, name, value) \
40 do { \
41 if ((dir) == EDMA_DIR_WRITE) \
42 SET_32(dw, wr_##name, value); \
43 else \
44 SET_32(dw, rd_##name, value); \
45 } while (0)
46
47 #define GET_RW_32(dw, dir, name) \
48 ((dir) == EDMA_DIR_WRITE \
49 ? GET_32(dw, wr_##name) \
50 : GET_32(dw, rd_##name))
51
52 #define SET_BOTH_32(dw, name, value) \
53 do { \
54 SET_32(dw, wr_##name, value); \
55 SET_32(dw, rd_##name, value); \
56 } while (0)
57
58 #define SET_64(dw, name, value) \
59 writeq(value, &(__dw_regs(dw)->name))
60
61 #define GET_64(dw, name) \
62 readq(&(__dw_regs(dw)->name))
63
64 #define SET_RW_64(dw, dir, name, value) \
65 do { \
66 if ((dir) == EDMA_DIR_WRITE) \
67 SET_64(dw, wr_##name, value); \
68 else \
69 SET_64(dw, rd_##name, value); \
70 } while (0)
71
72 #define GET_RW_64(dw, dir, name) \
73 ((dir) == EDMA_DIR_WRITE \
74 ? GET_64(dw, wr_##name) \
75 : GET_64(dw, rd_##name))
76
77 #define SET_BOTH_64(dw, name, value) \
78 do { \
79 SET_64(dw, wr_##name, value); \
80 SET_64(dw, rd_##name, value); \
81 } while (0)
82
83 #define SET_COMPAT(dw, name, value) \
84 writel(value, &(__dw_regs(dw)->type.unroll.name))
85
86 #define SET_RW_COMPAT(dw, dir, name, value) \
87 do { \
88 if ((dir) == EDMA_DIR_WRITE) \
89 SET_COMPAT(dw, wr_##name, value); \
90 else \
91 SET_COMPAT(dw, rd_##name, value); \
92 } while (0)
93
94 static inline struct dw_edma_v0_ch_regs __iomem *
__dw_ch_regs(struct dw_edma * dw,enum dw_edma_dir dir,u16 ch)95 __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
96 {
97 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY)
98 return &(__dw_regs(dw)->type.legacy.ch);
99
100 if (dir == EDMA_DIR_WRITE)
101 return &__dw_regs(dw)->type.unroll.ch[ch].wr;
102
103 return &__dw_regs(dw)->type.unroll.ch[ch].rd;
104 }
105
writel_ch(struct dw_edma * dw,enum dw_edma_dir dir,u16 ch,u32 value,void __iomem * addr)106 static inline void writel_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
107 u32 value, void __iomem *addr)
108 {
109 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) {
110 u32 viewport_sel;
111 unsigned long flags;
112
113 raw_spin_lock_irqsave(&dw->lock, flags);
114
115 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
116 if (dir == EDMA_DIR_READ)
117 viewport_sel |= BIT(31);
118
119 writel(viewport_sel,
120 &(__dw_regs(dw)->type.legacy.viewport_sel));
121 writel(value, addr);
122
123 raw_spin_unlock_irqrestore(&dw->lock, flags);
124 } else {
125 writel(value, addr);
126 }
127 }
128
readl_ch(struct dw_edma * dw,enum dw_edma_dir dir,u16 ch,const void __iomem * addr)129 static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
130 const void __iomem *addr)
131 {
132 u32 value;
133
134 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) {
135 u32 viewport_sel;
136 unsigned long flags;
137
138 raw_spin_lock_irqsave(&dw->lock, flags);
139
140 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
141 if (dir == EDMA_DIR_READ)
142 viewport_sel |= BIT(31);
143
144 writel(viewport_sel,
145 &(__dw_regs(dw)->type.legacy.viewport_sel));
146 value = readl(addr);
147
148 raw_spin_unlock_irqrestore(&dw->lock, flags);
149 } else {
150 value = readl(addr);
151 }
152
153 return value;
154 }
155
156 #define SET_CH_32(dw, dir, ch, name, value) \
157 writel_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name))
158
159 #define GET_CH_32(dw, dir, ch, name) \
160 readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name))
161
writeq_ch(struct dw_edma * dw,enum dw_edma_dir dir,u16 ch,u64 value,void __iomem * addr)162 static inline void writeq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
163 u64 value, void __iomem *addr)
164 {
165 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) {
166 u32 viewport_sel;
167 unsigned long flags;
168
169 raw_spin_lock_irqsave(&dw->lock, flags);
170
171 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
172 if (dir == EDMA_DIR_READ)
173 viewport_sel |= BIT(31);
174
175 writel(viewport_sel,
176 &(__dw_regs(dw)->type.legacy.viewport_sel));
177 writeq(value, addr);
178
179 raw_spin_unlock_irqrestore(&dw->lock, flags);
180 } else {
181 writeq(value, addr);
182 }
183 }
184
readq_ch(struct dw_edma * dw,enum dw_edma_dir dir,u16 ch,const void __iomem * addr)185 static inline u64 readq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
186 const void __iomem *addr)
187 {
188 u64 value;
189
190 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) {
191 u32 viewport_sel;
192 unsigned long flags;
193
194 raw_spin_lock_irqsave(&dw->lock, flags);
195
196 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
197 if (dir == EDMA_DIR_READ)
198 viewport_sel |= BIT(31);
199
200 writel(viewport_sel,
201 &(__dw_regs(dw)->type.legacy.viewport_sel));
202 value = readq(addr);
203
204 raw_spin_unlock_irqrestore(&dw->lock, flags);
205 } else {
206 value = readq(addr);
207 }
208
209 return value;
210 }
211
212 #define SET_CH_64(dw, dir, ch, name, value) \
213 writeq_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name))
214
215 #define GET_CH_64(dw, dir, ch, name) \
216 readq_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name))
217
218 /* eDMA management callbacks */
dw_edma_v0_core_off(struct dw_edma * dw)219 void dw_edma_v0_core_off(struct dw_edma *dw)
220 {
221 SET_BOTH_32(dw, int_mask,
222 EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
223 SET_BOTH_32(dw, int_clear,
224 EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
225 SET_BOTH_32(dw, engine_en, 0);
226 }
227
dw_edma_v0_core_ch_count(struct dw_edma * dw,enum dw_edma_dir dir)228 u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
229 {
230 u32 num_ch;
231
232 if (dir == EDMA_DIR_WRITE)
233 num_ch = FIELD_GET(EDMA_V0_WRITE_CH_COUNT_MASK,
234 GET_32(dw, ctrl));
235 else
236 num_ch = FIELD_GET(EDMA_V0_READ_CH_COUNT_MASK,
237 GET_32(dw, ctrl));
238
239 if (num_ch > EDMA_V0_MAX_NR_CH)
240 num_ch = EDMA_V0_MAX_NR_CH;
241
242 return (u16)num_ch;
243 }
244
dw_edma_v0_core_ch_status(struct dw_edma_chan * chan)245 enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan)
246 {
247 struct dw_edma *dw = chan->dw;
248 u32 tmp;
249
250 tmp = FIELD_GET(EDMA_V0_CH_STATUS_MASK,
251 GET_CH_32(dw, chan->dir, chan->id, ch_control1));
252
253 if (tmp == 1)
254 return DMA_IN_PROGRESS;
255 else if (tmp == 3)
256 return DMA_COMPLETE;
257 else
258 return DMA_ERROR;
259 }
260
dw_edma_v0_core_clear_done_int(struct dw_edma_chan * chan)261 void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan)
262 {
263 struct dw_edma *dw = chan->dw;
264
265 SET_RW_32(dw, chan->dir, int_clear,
266 FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)));
267 }
268
dw_edma_v0_core_clear_abort_int(struct dw_edma_chan * chan)269 void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
270 {
271 struct dw_edma *dw = chan->dw;
272
273 SET_RW_32(dw, chan->dir, int_clear,
274 FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)));
275 }
276
dw_edma_v0_core_status_done_int(struct dw_edma * dw,enum dw_edma_dir dir)277 u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir)
278 {
279 return FIELD_GET(EDMA_V0_DONE_INT_MASK,
280 GET_RW_32(dw, dir, int_status));
281 }
282
dw_edma_v0_core_status_abort_int(struct dw_edma * dw,enum dw_edma_dir dir)283 u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir)
284 {
285 return FIELD_GET(EDMA_V0_ABORT_INT_MASK,
286 GET_RW_32(dw, dir, int_status));
287 }
288
dw_edma_v0_write_ll_data(struct dw_edma_chunk * chunk,int i,u32 control,u32 size,u64 sar,u64 dar)289 static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i,
290 u32 control, u32 size, u64 sar, u64 dar)
291 {
292 ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);
293
294 if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
295 struct dw_edma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;
296
297 lli->control = control;
298 lli->transfer_size = size;
299 lli->sar.reg = sar;
300 lli->dar.reg = dar;
301 } else {
302 struct dw_edma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;
303
304 writel(control, &lli->control);
305 writel(size, &lli->transfer_size);
306 writeq(sar, &lli->sar.reg);
307 writeq(dar, &lli->dar.reg);
308 }
309 }
310
dw_edma_v0_write_ll_link(struct dw_edma_chunk * chunk,int i,u32 control,u64 pointer)311 static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk,
312 int i, u32 control, u64 pointer)
313 {
314 ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);
315
316 if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
317 struct dw_edma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;
318
319 llp->control = control;
320 llp->llp.reg = pointer;
321 } else {
322 struct dw_edma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;
323
324 writel(control, &llp->control);
325 writeq(pointer, &llp->llp.reg);
326 }
327 }
328
dw_edma_v0_core_write_chunk(struct dw_edma_chunk * chunk)329 static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
330 {
331 struct dw_edma_burst *child;
332 struct dw_edma_chan *chan = chunk->chan;
333 u32 control = 0, i = 0;
334 int j;
335
336 if (chunk->cb)
337 control = DW_EDMA_V0_CB;
338
339 j = chunk->bursts_alloc;
340 list_for_each_entry(child, &chunk->burst->list, list) {
341 j--;
342 if (!j) {
343 control |= DW_EDMA_V0_LIE;
344 if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
345 control |= DW_EDMA_V0_RIE;
346 }
347
348 dw_edma_v0_write_ll_data(chunk, i++, control, child->sz,
349 child->sar, child->dar);
350 }
351
352 control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB;
353 if (!chunk->cb)
354 control |= DW_EDMA_V0_CB;
355
356 dw_edma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);
357 }
358
dw_edma_v0_core_start(struct dw_edma_chunk * chunk,bool first)359 void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
360 {
361 struct dw_edma_chan *chan = chunk->chan;
362 struct dw_edma *dw = chan->dw;
363 u32 tmp;
364
365 dw_edma_v0_core_write_chunk(chunk);
366
367 if (first) {
368 /* Enable engine */
369 SET_RW_32(dw, chan->dir, engine_en, BIT(0));
370 if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) {
371 switch (chan->id) {
372 case 0:
373 SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en,
374 BIT(0));
375 break;
376 case 1:
377 SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en,
378 BIT(0));
379 break;
380 case 2:
381 SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en,
382 BIT(0));
383 break;
384 case 3:
385 SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en,
386 BIT(0));
387 break;
388 case 4:
389 SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en,
390 BIT(0));
391 break;
392 case 5:
393 SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en,
394 BIT(0));
395 break;
396 case 6:
397 SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en,
398 BIT(0));
399 break;
400 case 7:
401 SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en,
402 BIT(0));
403 break;
404 }
405 }
406 /* Interrupt unmask - done, abort */
407 tmp = GET_RW_32(dw, chan->dir, int_mask);
408 tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id));
409 tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id));
410 SET_RW_32(dw, chan->dir, int_mask, tmp);
411 /* Linked list error */
412 tmp = GET_RW_32(dw, chan->dir, linked_list_err_en);
413 tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id));
414 SET_RW_32(dw, chan->dir, linked_list_err_en, tmp);
415 /* Channel control */
416 SET_CH_32(dw, chan->dir, chan->id, ch_control1,
417 (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE));
418 /* Linked list */
419 /* llp is not aligned on 64bit -> keep 32bit accesses */
420 SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
421 lower_32_bits(chunk->ll_region.paddr));
422 SET_CH_32(dw, chan->dir, chan->id, llp.msb,
423 upper_32_bits(chunk->ll_region.paddr));
424 }
425 /* Doorbell */
426 SET_RW_32(dw, chan->dir, doorbell,
427 FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
428 }
429
dw_edma_v0_core_device_config(struct dw_edma_chan * chan)430 int dw_edma_v0_core_device_config(struct dw_edma_chan *chan)
431 {
432 struct dw_edma *dw = chan->dw;
433 u32 tmp = 0;
434
435 /* MSI done addr - low, high */
436 SET_RW_32(dw, chan->dir, done_imwr.lsb, chan->msi.address_lo);
437 SET_RW_32(dw, chan->dir, done_imwr.msb, chan->msi.address_hi);
438 /* MSI abort addr - low, high */
439 SET_RW_32(dw, chan->dir, abort_imwr.lsb, chan->msi.address_lo);
440 SET_RW_32(dw, chan->dir, abort_imwr.msb, chan->msi.address_hi);
441 /* MSI data - low, high */
442 switch (chan->id) {
443 case 0:
444 case 1:
445 tmp = GET_RW_32(dw, chan->dir, ch01_imwr_data);
446 break;
447
448 case 2:
449 case 3:
450 tmp = GET_RW_32(dw, chan->dir, ch23_imwr_data);
451 break;
452
453 case 4:
454 case 5:
455 tmp = GET_RW_32(dw, chan->dir, ch45_imwr_data);
456 break;
457
458 case 6:
459 case 7:
460 tmp = GET_RW_32(dw, chan->dir, ch67_imwr_data);
461 break;
462 }
463
464 if (chan->id & BIT(0)) {
465 /* Channel odd {1, 3, 5, 7} */
466 tmp &= EDMA_V0_CH_EVEN_MSI_DATA_MASK;
467 tmp |= FIELD_PREP(EDMA_V0_CH_ODD_MSI_DATA_MASK,
468 chan->msi.data);
469 } else {
470 /* Channel even {0, 2, 4, 6} */
471 tmp &= EDMA_V0_CH_ODD_MSI_DATA_MASK;
472 tmp |= FIELD_PREP(EDMA_V0_CH_EVEN_MSI_DATA_MASK,
473 chan->msi.data);
474 }
475
476 switch (chan->id) {
477 case 0:
478 case 1:
479 SET_RW_32(dw, chan->dir, ch01_imwr_data, tmp);
480 break;
481
482 case 2:
483 case 3:
484 SET_RW_32(dw, chan->dir, ch23_imwr_data, tmp);
485 break;
486
487 case 4:
488 case 5:
489 SET_RW_32(dw, chan->dir, ch45_imwr_data, tmp);
490 break;
491
492 case 6:
493 case 7:
494 SET_RW_32(dw, chan->dir, ch67_imwr_data, tmp);
495 break;
496 }
497
498 return 0;
499 }
500
501 /* eDMA debugfs callbacks */
dw_edma_v0_core_debugfs_on(struct dw_edma * dw)502 void dw_edma_v0_core_debugfs_on(struct dw_edma *dw)
503 {
504 dw_edma_v0_debugfs_on(dw);
505 }
506