1 // SPDX-License-Identifier: BSD-2-Clause
2 /*
3 * Cadence DDR Controller
4 *
5 * Copyright (C) 2015 Renesas Electronics Europe Ltd
6 */
7
8 /*
9 * The Cadence DDR Controller has a huge number of registers that principally
10 * cover two aspects, DDR specific timing information and AXI bus interfacing.
11 * Cadence's TCL script generates all of the register values for specific
12 * DDR devices operating at a specific frequency. The TCL script uses Denali
13 * SOMA files as inputs. The tool also generates the AXI bus register values as
14 * well, however this driver assumes that users will want to modifiy these to
15 * meet a specific application's needs.
16 * Therefore, this driver is passed two arrays containing register values for
17 * the DDR device specific information, and explicity sets the AXI registers.
18 *
19 * AXI bus interfacing:
20 * The controller has four AXI slaves connections, and each of these can be
21 * programmed to accept requests from specific AXI masters (using their IDs).
22 * The regions of DDR that can be accessed by each AXI slave can be set such
23 * as to isolate DDR used by one AXI master from another. Further, the maximum
24 * bandwidth allocated to each AXI slave can be set.
25 */
26
27 #include <linux/delay.h>
28 #include <linux/sizes.h>
29 #include <asm/io.h>
30 #include <wait_bit.h>
31 #include <renesas/ddr_ctrl.h>
32
33 /* avoid warning for real pr_debug in <linux/printk.h> */
34 #ifdef pr_debug
35 #undef pr_debug
36 #endif
37
38 #ifdef DEBUG
39 #define pr_debug(fmt, args...) printf(fmt, ##args)
40 #define pr_debug2(fmt, args...) printf(fmt, ##args)
41 #else
42 #define pr_debug(fmt, args...)
43 #define pr_debug2(fmt, args...)
44 #endif
45
46 #define DDR_NR_AXI_PORTS 4
47 #define DDR_NR_ENTRIES 16
48
49 #define DDR_START_REG (0) /* DENALI_CTL_00 */
50 #define DDR_CS0_MR1_REG (32 * 4) /* DENALI_CTL_32 */
51 #define DDR_CS0_MR2_REG (32 * 4 + 2) /* DENALI_CTL_32 */
52 #define DDR_CS1_MR1_REG (34 * 4 + 2) /* DENALI_CTL_34 */
53 #define DDR_CS1_MR2_REG (35 * 4) /* DENALI_CTL_35 */
54 #define DDR_ECC_ENABLE_REG (36 * 4 + 2) /* DENALI_CTL_36 */
55 #define DDR_ECC_DISABLE_W_UC_ERR_REG (37 * 4 + 2) /* DENALI_CTL_37 */
56 #define DDR_HALF_DATAPATH_REG (54 * 4) /* DENALI_CTL_54 */
57 #define DDR_INTERRUPT_STATUS (56 * 4) /* DENALI_CTL_56 */
58 #define DDR_INTERRUPT_ACK (57 * 4) /* DENALI_CTL_57 */
59 #define DDR_INTERRUPT_MASK (58 * 4) /* DENALI_CTL_58 */
60 #define DDR_CS0_ODT_MAP_REG (62 * 4 + 2) /* DENALI_CTL_62 */
61 #define DDR_CS1_ODT_MAP_REG (63 * 4) /* DENALI_CTL_63 */
62 #define DDR_ODT_TODTL_2CMD (63 * 4 + 2) /* DENALI_CTL_63 */
63 #define DDR_ODT_TODTH_WR (63 * 4 + 3) /* DENALI_CTL_63 */
64 #define DDR_ODT_TODTH_RD (64 * 4 + 0) /* DENALI_CTL_64 */
65 #define DDR_ODT_EN (64 * 4 + 1) /* DENALI_CTL_64 */
66 #define DDR_ODT_WR_TO_ODTH (64 * 4 + 2) /* DENALI_CTL_64 */
67 #define DDR_ODT_RD_TO_ODTH (64 * 4 + 3) /* DENALI_CTL_64 */
68 #define DDR_DIFF_CS_DELAY_REG (66 * 4) /* DENALI_CTL_66 */
69 #define DDR_SAME_CS_DELAY_REG (67 * 4) /* DENALI_CTL_67 */
70 #define DDR_RW_PRIORITY_REGS (87 * 4 + 2) /* DENALI_CTL_87 */
71 #define DDR_RW_FIFO_TYPE_REGS (88 * 4) /* DENALI_CTL_88 */
72 #define DDR_AXI_PORT_PROT_ENABLE_REG (90 * 4 + 3) /* DENALI_CTL_90 */
73 #define DDR_ADDR_RANGE_REGS (91 * 4) /* DENALI_CTL_91 */
74 #define DDR_RANGE_PROT_REGS (218 * 4 + 2) /* DENALI_CTL_218 */
75 #define DDR_ARB_CMD_Q_THRESHOLD_REG (346 * 4 + 2) /* DENALI_CTL_346 */
76 #define DDR_AXI_PORT_BANDWIDTH_REG (346 * 4 + 3) /* DENALI_CTL_346 */
77 #define DDR_OPT_RMODW_REG (372 * 4 + 3) /* DENALI_CTL_372 */
78
ddrc_writeb(u8 val,void * p)79 static void ddrc_writeb(u8 val, void *p)
80 {
81 pr_debug2("DDR: %p = 0x%02x\n", p, val);
82 writeb(val, p);
83 }
84
ddrc_writew(u16 val,void * p)85 static void ddrc_writew(u16 val, void *p)
86 {
87 pr_debug2("DDR: %p = 0x%04x\n", p, val);
88 writew(val, p);
89 }
90
ddrc_writel(u32 val,void * p)91 static void ddrc_writel(u32 val, void *p)
92 {
93 pr_debug2("DDR: %p = 0x%08x\n", p, val);
94 writel(val, p);
95 }
96
cdns_ddr_set_mr1(void * base,int cs,u16 odt_impedance,u16 drive_strength)97 void cdns_ddr_set_mr1(void *base, int cs, u16 odt_impedance, u16 drive_strength)
98 {
99 void *reg;
100 u16 tmp;
101
102 if (cs == 0)
103 reg = (u8 *)base + DDR_CS0_MR1_REG;
104 else
105 reg = (u8 *)base + DDR_CS1_MR1_REG;
106
107 tmp = readw(reg);
108
109 tmp &= ~MODE_REGISTER_MASK;
110 tmp |= MODE_REGISTER_MR1;
111
112 tmp &= ~MR1_ODT_IMPEDANCE_MASK;
113 tmp |= odt_impedance;
114
115 tmp &= ~MR1_DRIVE_STRENGTH_MASK;
116 tmp |= drive_strength;
117
118 writew(tmp, reg);
119 }
120
cdns_ddr_set_mr2(void * base,int cs,u16 dynamic_odt,u16 self_refresh_temp)121 void cdns_ddr_set_mr2(void *base, int cs, u16 dynamic_odt, u16 self_refresh_temp)
122 {
123 void *reg;
124 u16 tmp;
125
126 if (cs == 0)
127 reg = (u8 *)base + DDR_CS0_MR2_REG;
128 else
129 reg = (u8 *)base + DDR_CS1_MR2_REG;
130
131 tmp = readw(reg);
132
133 tmp &= ~MODE_REGISTER_MASK;
134 tmp |= MODE_REGISTER_MR2;
135
136 tmp &= ~MR2_DYNAMIC_ODT_MASK;
137 tmp |= dynamic_odt;
138
139 tmp &= ~MR2_SELF_REFRESH_TEMP_MASK;
140 tmp |= self_refresh_temp;
141
142 writew(tmp, reg);
143 }
144
cdns_ddr_set_odt_map(void * base,int cs,u16 odt_map)145 void cdns_ddr_set_odt_map(void *base, int cs, u16 odt_map)
146 {
147 void *reg;
148
149 if (cs == 0)
150 reg = (u8 *)base + DDR_CS0_ODT_MAP_REG;
151 else
152 reg = (u8 *)base + DDR_CS1_ODT_MAP_REG;
153
154 writew(odt_map, reg);
155 }
156
cdns_ddr_set_odt_times(void * base,u8 TODTL_2CMD,u8 TODTH_WR,u8 TODTH_RD,u8 WR_TO_ODTH,u8 RD_TO_ODTH)157 void cdns_ddr_set_odt_times(void *base, u8 TODTL_2CMD, u8 TODTH_WR, u8 TODTH_RD,
158 u8 WR_TO_ODTH, u8 RD_TO_ODTH)
159 {
160 writeb(TODTL_2CMD, (u8 *)base + DDR_ODT_TODTL_2CMD);
161 writeb(TODTH_WR, (u8 *)base + DDR_ODT_TODTH_WR);
162 writeb(TODTH_RD, (u8 *)base + DDR_ODT_TODTH_RD);
163 writeb(1, (u8 *)base + DDR_ODT_EN);
164 writeb(WR_TO_ODTH, (u8 *)base + DDR_ODT_WR_TO_ODTH);
165 writeb(RD_TO_ODTH, (u8 *)base + DDR_ODT_RD_TO_ODTH);
166 }
167
cdns_ddr_set_same_cs_delays(void * base,u8 r2r,u8 r2w,u8 w2r,u8 w2w)168 void cdns_ddr_set_same_cs_delays(void *base, u8 r2r, u8 r2w, u8 w2r, u8 w2w)
169 {
170 u32 val = (w2w << 24) | (w2r << 16) | (r2w << 8) | r2r;
171
172 writel(val, (u8 *)base + DDR_SAME_CS_DELAY_REG);
173 }
174
cdns_ddr_set_diff_cs_delays(void * base,u8 r2r,u8 r2w,u8 w2r,u8 w2w)175 void cdns_ddr_set_diff_cs_delays(void *base, u8 r2r, u8 r2w, u8 w2r, u8 w2w)
176 {
177 u32 val = (w2w << 24) | (w2r << 16) | (r2w << 8) | r2r;
178
179 writel(val, (u8 *)base + DDR_DIFF_CS_DELAY_REG);
180 }
181
cdns_ddr_set_port_rw_priority(void * base,int port,u8 read_pri,u8 write_pri)182 void cdns_ddr_set_port_rw_priority(void *base, int port,
183 u8 read_pri, u8 write_pri)
184 {
185 u8 *reg8 = (u8 *)base + DDR_RW_PRIORITY_REGS;
186
187 reg8 += (port * 3);
188 pr_debug("%s port %d (reg8=%p, DENALI_CTL_%d)\n",
189 __func__, port, reg8, (reg8 - (u8 *)base) / 4);
190
191 ddrc_writeb(read_pri, reg8++);
192 ddrc_writeb(write_pri, reg8++);
193 }
194
195 /* The DDR Controller has 16 entries. Each entry can specify an allowed address
196 * range (with 16KB resolution) for one of the 4 AXI slave ports.
197 */
cdns_ddr_enable_port_addr_range(void * base,int port,int entry,u32 addr_start,u32 size)198 void cdns_ddr_enable_port_addr_range(void *base, int port, int entry,
199 u32 addr_start, u32 size)
200 {
201 u32 addr_end;
202 u32 *reg32 = (u32 *)((u8 *)base + DDR_ADDR_RANGE_REGS);
203 u32 tmp;
204
205 reg32 += (port * DDR_NR_ENTRIES * 2);
206 reg32 += (entry * 2);
207 pr_debug("%s port %d, entry %d (reg32=%p, DENALI_CTL_%d)\n",
208 __func__, port, entry, reg32, ((u8 *)reg32 - (u8 *)base) / 4);
209
210 /* These registers represent 16KB address blocks */
211 addr_start /= SZ_16K;
212 size /= SZ_16K;
213 if (size)
214 addr_end = addr_start + size - 1;
215 else
216 addr_end = addr_start;
217
218 ddrc_writel(addr_start, reg32++);
219
220 /*
221 * end_addr: Ensure we only set the bottom 18-bits as DENALI_CTL_218
222 * also contains the AXI0 range protection bits.
223 */
224 tmp = readl(reg32);
225 tmp &= ~(BIT(18) - 1);
226 tmp |= addr_end;
227 ddrc_writel(tmp, reg32);
228 }
229
cdns_ddr_enable_addr_range(void * base,int entry,u32 addr_start,u32 size)230 void cdns_ddr_enable_addr_range(void *base, int entry,
231 u32 addr_start, u32 size)
232 {
233 int axi;
234
235 for (axi = 0; axi < DDR_NR_AXI_PORTS; axi++)
236 cdns_ddr_enable_port_addr_range(base, axi, entry,
237 addr_start, size);
238 }
239
cdns_ddr_enable_port_prot(void * base,int port,int entry,enum cdns_ddr_range_prot range_protection_bits,u16 range_RID_check_bits,u16 range_WID_check_bits,u8 range_RID_check_bits_ID_lookup,u8 range_WID_check_bits_ID_lookup)240 void cdns_ddr_enable_port_prot(void *base, int port, int entry,
241 enum cdns_ddr_range_prot range_protection_bits,
242 u16 range_RID_check_bits,
243 u16 range_WID_check_bits,
244 u8 range_RID_check_bits_ID_lookup,
245 u8 range_WID_check_bits_ID_lookup)
246 {
247 /*
248 * Technically, the offset here points to the byte before the start of
249 * the range protection registers. However, all entries consist of 8
250 * bytes, except the first one (which is missing a padding byte) so we
251 * work around that subtlely.
252 */
253 u8 *reg8 = (u8 *)base + DDR_RANGE_PROT_REGS;
254
255 reg8 += (port * DDR_NR_ENTRIES * 8);
256 reg8 += (entry * 8);
257 pr_debug("%s port %d, entry %d (reg8=%p, DENALI_CTL_%d)\n",
258 __func__, port, entry, reg8, (reg8 - (u8 *)base) / 4);
259
260 if (port == 0 && entry == 0)
261 ddrc_writeb(range_protection_bits, reg8 + 1);
262 else
263 ddrc_writeb(range_protection_bits, reg8);
264
265 ddrc_writew(range_RID_check_bits, reg8 + 2);
266 ddrc_writew(range_WID_check_bits, reg8 + 4);
267 ddrc_writeb(range_RID_check_bits_ID_lookup, reg8 + 6);
268 ddrc_writeb(range_WID_check_bits_ID_lookup, reg8 + 7);
269 }
270
cdns_ddr_enable_prot(void * base,int entry,enum cdns_ddr_range_prot range_protection_bits,u16 range_RID_check_bits,u16 range_WID_check_bits,u8 range_RID_check_bits_ID_lookup,u8 range_WID_check_bits_ID_lookup)271 void cdns_ddr_enable_prot(void *base, int entry,
272 enum cdns_ddr_range_prot range_protection_bits,
273 u16 range_RID_check_bits,
274 u16 range_WID_check_bits,
275 u8 range_RID_check_bits_ID_lookup,
276 u8 range_WID_check_bits_ID_lookup)
277 {
278 int axi;
279
280 for (axi = 0; axi < DDR_NR_AXI_PORTS; axi++)
281 cdns_ddr_enable_port_prot(base, axi, entry,
282 range_protection_bits,
283 range_RID_check_bits,
284 range_WID_check_bits,
285 range_RID_check_bits_ID_lookup,
286 range_WID_check_bits_ID_lookup);
287 }
288
cdns_ddr_set_port_bandwidth(void * base,int port,u8 max_percent,u8 overflow_ok)289 void cdns_ddr_set_port_bandwidth(void *base, int port,
290 u8 max_percent, u8 overflow_ok)
291 {
292 u8 *reg8 = (u8 *)base + DDR_AXI_PORT_BANDWIDTH_REG;
293
294 reg8 += (port * 3);
295 pr_debug("%s port %d, (reg8=%p, DENALI_CTL_%d)\n",
296 __func__, port, reg8, (reg8 - (u8 *)base) / 4);
297
298 ddrc_writeb(max_percent, reg8++); /* Maximum bandwidth percentage */
299 ddrc_writeb(overflow_ok, reg8++); /* Bandwidth overflow allowed */
300 }
301
cdns_ddr_ctrl_init(void * ddr_ctrl_basex,int async,const u32 * reg0,const u32 * reg350,u32 ddr_start_addr,u32 ddr_size,int enable_ecc,int enable_8bit)302 void cdns_ddr_ctrl_init(void *ddr_ctrl_basex, int async,
303 const u32 *reg0, const u32 *reg350,
304 u32 ddr_start_addr, u32 ddr_size,
305 int enable_ecc, int enable_8bit)
306 {
307 int i, axi, entry;
308 u32 *ddr_ctrl_base = (u32 *)ddr_ctrl_basex;
309 u8 *base8 = (u8 *)ddr_ctrl_basex;
310
311 ddrc_writel(*reg0, ddr_ctrl_base + 0);
312 /* 1 to 6 are read only */
313 for (i = 7; i <= 26; i++)
314 ddrc_writel(*(reg0 + i), ddr_ctrl_base + i);
315 /* 27 to 29 are not changed */
316 for (i = 30; i <= 87; i++)
317 ddrc_writel(*(reg0 + i), ddr_ctrl_base + i);
318
319 /* Enable/disable ECC */
320 if (enable_ecc) {
321 pr_debug("%s enabling ECC\n", __func__);
322 ddrc_writeb(1, base8 + DDR_ECC_ENABLE_REG);
323 } else {
324 ddrc_writeb(0, base8 + DDR_ECC_ENABLE_REG);
325 }
326
327 /* ECC: Disable corruption for read/modify/write operations */
328 ddrc_writeb(1, base8 + DDR_ECC_DISABLE_W_UC_ERR_REG);
329
330 /* Set 8/16-bit data width using reduce bit (enable half datapath)*/
331 if (enable_8bit) {
332 pr_debug("%s using 8-bit data\n", __func__);
333 ddrc_writeb(1, base8 + DDR_HALF_DATAPATH_REG);
334 } else {
335 ddrc_writeb(0, base8 + DDR_HALF_DATAPATH_REG);
336 }
337
338 /* Threshold for command queue */
339 ddrc_writeb(4, base8 + DDR_ARB_CMD_Q_THRESHOLD_REG);
340
341 /* AXI port protection => enable */
342 ddrc_writeb(0x01, base8 + DDR_AXI_PORT_PROT_ENABLE_REG);
343
344 /* Set port interface type, default port priority and bandwidths */
345 for (axi = 0; axi < DDR_NR_AXI_PORTS; axi++) {
346 /* port interface type: synchronous or asynchronous AXI clock */
347 u8 *fifo_reg = base8 + DDR_RW_FIFO_TYPE_REGS + (axi * 3);
348
349 if (async)
350 ddrc_writeb(0, fifo_reg);
351 else
352 ddrc_writeb(3, fifo_reg);
353
354 /* R/W priorities */
355 cdns_ddr_set_port_rw_priority(ddr_ctrl_base, axi, 2, 2);
356
357 /* AXI bandwidth */
358 cdns_ddr_set_port_bandwidth(ddr_ctrl_base, axi, 50, 1);
359 }
360
361 /*
362 * The hardware requires that the valid address ranges must not overlap.
363 * So, we initialise all address ranges to be above the DDR, length 0.
364 */
365 for (entry = 0; entry < DDR_NR_ENTRIES; entry++)
366 cdns_ddr_enable_addr_range(ddr_ctrl_base, entry,
367 ddr_start_addr + ddr_size, 0);
368
369 for (i = 350; i <= 374; i++)
370 ddrc_writel(*(reg350 - 350 + i), ddr_ctrl_base + i);
371
372 /* Disable optimised read-modify-write logic */
373 ddrc_writeb(0, base8 + DDR_OPT_RMODW_REG);
374
375 /*
376 * Disable all interrupts, we are not handling them.
377 * For detail of the interrupt mask, ack and status bits, see the
378 * manual's description of the 'int_status' parameter.
379 */
380 ddrc_writel(0, base8 + DDR_INTERRUPT_MASK);
381
382 /*
383 * Default settings to enable full access to the entire DDR.
384 * Users can set different ranges and access rights by calling these
385 * functions before calling cdns_ddr_ctrl_start().
386 */
387 cdns_ddr_enable_addr_range(ddr_ctrl_base, 0,
388 ddr_start_addr, ddr_size);
389 cdns_ddr_enable_prot(ddr_ctrl_base, 0, CDNS_DDR_RANGE_PROT_BITS_FULL,
390 0xffff, 0xffff, 0x0f, 0x0f);
391 }
392
cdns_ddr_ctrl_start(void * ddr_ctrl_basex)393 void cdns_ddr_ctrl_start(void *ddr_ctrl_basex)
394 {
395 u32 *ddr_ctrl_base = (u32 *)ddr_ctrl_basex;
396 u8 *base8 = (u8 *)ddr_ctrl_basex;
397
398 /* Start */
399 ddrc_writeb(1, base8 + DDR_START_REG);
400
401 /* Wait for controller to be ready (interrupt status) */
402 wait_for_bit_le32(base8 + DDR_INTERRUPT_STATUS, 0x100, true, 1000, false);
403
404 /* clear all interrupts */
405 ddrc_writel(~0, base8 + DDR_INTERRUPT_ACK);
406
407 /* Step 19 Wait 500us from MRESETB=1 */
408 udelay(500);
409
410 /* Step 20 tCKSRX wait (From supply stable clock for MCK) */
411 /* DENALI_CTL_19 TREF_ENABLE=0x1(=1), AREFRESH=0x1(=1) */
412 ddrc_writel(0x01000100, ddr_ctrl_base + 19);
413 }
414