1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "i915_drv.h"
7 
8 #include "intel_gt_mcr.h"
9 #include "intel_gt_print.h"
10 #include "intel_gt_regs.h"
11 
12 /**
13  * DOC: GT Multicast/Replicated (MCR) Register Support
14  *
15  * Some GT registers are designed as "multicast" or "replicated" registers:
16  * multiple instances of the same register share a single MMIO offset.  MCR
17  * registers are generally used when the hardware needs to potentially track
18  * independent values of a register per hardware unit (e.g., per-subslice,
19  * per-L3bank, etc.).  The specific types of replication that exist vary
20  * per-platform.
21  *
22  * MMIO accesses to MCR registers are controlled according to the settings
23  * programmed in the platform's MCR_SELECTOR register(s).  MMIO writes to MCR
24  * registers can be done in either a (i.e., a single write updates all
25  * instances of the register to the same value) or unicast (a write updates only
26  * one specific instance).  Reads of MCR registers always operate in a unicast
27  * manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR.
28  * Selection of a specific MCR instance for unicast operations is referred to
29  * as "steering."
30  *
31  * If MCR register operations are steered toward a hardware unit that is
32  * fused off or currently powered down due to power gating, the MMIO operation
33  * is "terminated" by the hardware.  Terminated read operations will return a
34  * value of zero and terminated unicast write operations will be silently
35  * ignored.
36  */
37 
38 #define HAS_MSLICE_STEERING(dev_priv)	(INTEL_INFO(dev_priv)->has_mslice_steering)
39 
40 static const char * const intel_steering_types[] = {
41 	"L3BANK",
42 	"MSLICE",
43 	"LNCF",
44 	"GAM",
45 	"DSS",
46 	"OADDRM",
47 	"INSTANCE 0",
48 };
49 
50 static const struct intel_mmio_range icl_l3bank_steering_table[] = {
51 	{ 0x00B100, 0x00B3FF },
52 	{},
53 };
54 
55 /*
56  * Although the bspec lists more "MSLICE" ranges than shown here, some of those
57  * are of a "GAM" subclass that has special rules.  Thus we use a separate
58  * GAM table farther down for those.
59  */
60 static const struct intel_mmio_range xehpsdv_mslice_steering_table[] = {
61 	{ 0x00DD00, 0x00DDFF },
62 	{ 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */
63 	{},
64 };
65 
66 static const struct intel_mmio_range xehpsdv_gam_steering_table[] = {
67 	{ 0x004000, 0x004AFF },
68 	{ 0x00C800, 0x00CFFF },
69 	{},
70 };
71 
72 static const struct intel_mmio_range xehpsdv_lncf_steering_table[] = {
73 	{ 0x00B000, 0x00B0FF },
74 	{ 0x00D800, 0x00D8FF },
75 	{},
76 };
77 
78 static const struct intel_mmio_range dg2_lncf_steering_table[] = {
79 	{ 0x00B000, 0x00B0FF },
80 	{ 0x00D880, 0x00D8FF },
81 	{},
82 };
83 
84 /*
85  * We have several types of MCR registers on PVC where steering to (0,0)
86  * will always provide us with a non-terminated value.  We'll stick them
87  * all in the same table for simplicity.
88  */
89 static const struct intel_mmio_range pvc_instance0_steering_table[] = {
90 	{ 0x004000, 0x004AFF },		/* HALF-BSLICE */
91 	{ 0x008800, 0x00887F },		/* CC */
92 	{ 0x008A80, 0x008AFF },		/* TILEPSMI */
93 	{ 0x00B000, 0x00B0FF },		/* HALF-BSLICE */
94 	{ 0x00B100, 0x00B3FF },		/* L3BANK */
95 	{ 0x00C800, 0x00CFFF },		/* HALF-BSLICE */
96 	{ 0x00D800, 0x00D8FF },		/* HALF-BSLICE */
97 	{ 0x00DD00, 0x00DDFF },		/* BSLICE */
98 	{ 0x00E900, 0x00E9FF },		/* HALF-BSLICE */
99 	{ 0x00EC00, 0x00EEFF },		/* HALF-BSLICE */
100 	{ 0x00F000, 0x00FFFF },		/* HALF-BSLICE */
101 	{ 0x024180, 0x0241FF },		/* HALF-BSLICE */
102 	{},
103 };
104 
105 static const struct intel_mmio_range xelpg_instance0_steering_table[] = {
106 	{ 0x000B00, 0x000BFF },         /* SQIDI */
107 	{ 0x001000, 0x001FFF },         /* SQIDI */
108 	{ 0x004000, 0x0048FF },         /* GAM */
109 	{ 0x008700, 0x0087FF },         /* SQIDI */
110 	{ 0x00B000, 0x00B0FF },         /* NODE */
111 	{ 0x00C800, 0x00CFFF },         /* GAM */
112 	{ 0x00D880, 0x00D8FF },         /* NODE */
113 	{ 0x00DD00, 0x00DDFF },         /* OAAL2 */
114 	{},
115 };
116 
117 static const struct intel_mmio_range xelpg_l3bank_steering_table[] = {
118 	{ 0x00B100, 0x00B3FF },
119 	{},
120 };
121 
122 /* DSS steering is used for SLICE ranges as well */
123 static const struct intel_mmio_range xelpg_dss_steering_table[] = {
124 	{ 0x005200, 0x0052FF },		/* SLICE */
125 	{ 0x005500, 0x007FFF },		/* SLICE */
126 	{ 0x008140, 0x00815F },		/* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
127 	{ 0x0094D0, 0x00955F },		/* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
128 	{ 0x009680, 0x0096FF },		/* DSS */
129 	{ 0x00D800, 0x00D87F },		/* SLICE */
130 	{ 0x00DC00, 0x00DCFF },		/* SLICE */
131 	{ 0x00DE80, 0x00E8FF },		/* DSS (0xE000-0xE0FF reserved) */
132 	{},
133 };
134 
135 static const struct intel_mmio_range xelpmp_oaddrm_steering_table[] = {
136 	{ 0x393200, 0x39323F },
137 	{ 0x393400, 0x3934FF },
138 	{},
139 };
140 
intel_gt_mcr_init(struct intel_gt * gt)141 void intel_gt_mcr_init(struct intel_gt *gt)
142 {
143 	struct drm_i915_private *i915 = gt->i915;
144 	unsigned long fuse;
145 	int i;
146 
147 	spin_lock_init(&gt->mcr_lock);
148 
149 	/*
150 	 * An mslice is unavailable only if both the meml3 for the slice is
151 	 * disabled *and* all of the DSS in the slice (quadrant) are disabled.
152 	 */
153 	if (HAS_MSLICE_STEERING(i915)) {
154 		gt->info.mslice_mask =
155 			intel_slicemask_from_xehp_dssmask(gt->info.sseu.subslice_mask,
156 							  GEN_DSS_PER_MSLICE);
157 		gt->info.mslice_mask |=
158 			(intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &
159 			 GEN12_MEML3_EN_MASK);
160 
161 		if (!gt->info.mslice_mask) /* should be impossible! */
162 			gt_warn(gt, "mslice mask all zero!\n");
163 	}
164 
165 	if (MEDIA_VER(i915) >= 13 && gt->type == GT_MEDIA) {
166 		gt->steering_table[OADDRM] = xelpmp_oaddrm_steering_table;
167 	} else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
168 		/* Wa_14016747170 */
169 		if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_A0, STEP_B0) ||
170 		    IS_MTL_GRAPHICS_STEP(i915, P, STEP_A0, STEP_B0))
171 			fuse = REG_FIELD_GET(MTL_GT_L3_EXC_MASK,
172 					     intel_uncore_read(gt->uncore,
173 							       MTL_GT_ACTIVITY_FACTOR));
174 		else
175 			fuse = REG_FIELD_GET(GT_L3_EXC_MASK,
176 					     intel_uncore_read(gt->uncore, XEHP_FUSE4));
177 
178 		/*
179 		 * Despite the register field being named "exclude mask" the
180 		 * bits actually represent enabled banks (two banks per bit).
181 		 */
182 		for_each_set_bit(i, &fuse, 3)
183 			gt->info.l3bank_mask |= 0x3 << 2 * i;
184 
185 		gt->steering_table[INSTANCE0] = xelpg_instance0_steering_table;
186 		gt->steering_table[L3BANK] = xelpg_l3bank_steering_table;
187 		gt->steering_table[DSS] = xelpg_dss_steering_table;
188 	} else if (IS_PONTEVECCHIO(i915)) {
189 		gt->steering_table[INSTANCE0] = pvc_instance0_steering_table;
190 	} else if (IS_DG2(i915)) {
191 		gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table;
192 		gt->steering_table[LNCF] = dg2_lncf_steering_table;
193 		/*
194 		 * No need to hook up the GAM table since it has a dedicated
195 		 * steering control register on DG2 and can use implicit
196 		 * steering.
197 		 */
198 	} else if (IS_XEHPSDV(i915)) {
199 		gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table;
200 		gt->steering_table[LNCF] = xehpsdv_lncf_steering_table;
201 		gt->steering_table[GAM] = xehpsdv_gam_steering_table;
202 	} else if (GRAPHICS_VER(i915) >= 11 &&
203 		   GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) {
204 		gt->steering_table[L3BANK] = icl_l3bank_steering_table;
205 		gt->info.l3bank_mask =
206 			~intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &
207 			GEN10_L3BANK_MASK;
208 		if (!gt->info.l3bank_mask) /* should be impossible! */
209 			gt_warn(gt, "L3 bank mask is all zero!\n");
210 	} else if (GRAPHICS_VER(i915) >= 11) {
211 		/*
212 		 * We expect all modern platforms to have at least some
213 		 * type of steering that needs to be initialized.
214 		 */
215 		MISSING_CASE(INTEL_INFO(i915)->platform);
216 	}
217 }
218 
219 /*
220  * Although the rest of the driver should use MCR-specific functions to
221  * read/write MCR registers, we still use the regular intel_uncore_* functions
222  * internally to implement those, so we need a way for the functions in this
223  * file to "cast" an i915_mcr_reg_t into an i915_reg_t.
224  */
mcr_reg_cast(const i915_mcr_reg_t mcr)225 static i915_reg_t mcr_reg_cast(const i915_mcr_reg_t mcr)
226 {
227 	i915_reg_t r = { .reg = mcr.reg };
228 
229 	return r;
230 }
231 
232 /*
233  * rw_with_mcr_steering_fw - Access a register with specific MCR steering
234  * @gt: GT to read register from
235  * @reg: register being accessed
236  * @rw_flag: FW_REG_READ for read access or FW_REG_WRITE for write access
237  * @group: group number (documented as "sliceid" on older platforms)
238  * @instance: instance number (documented as "subsliceid" on older platforms)
239  * @value: register value to be written (ignored for read)
240  *
241  * Context: The caller must hold the MCR lock
242  * Return: 0 for write access. register value for read access.
243  *
244  * Caller needs to make sure the relevant forcewake wells are up.
245  */
rw_with_mcr_steering_fw(struct intel_gt * gt,i915_mcr_reg_t reg,u8 rw_flag,int group,int instance,u32 value)246 static u32 rw_with_mcr_steering_fw(struct intel_gt *gt,
247 				   i915_mcr_reg_t reg, u8 rw_flag,
248 				   int group, int instance, u32 value)
249 {
250 	struct intel_uncore *uncore = gt->uncore;
251 	u32 mcr_mask, mcr_ss, mcr, old_mcr, val = 0;
252 
253 	lockdep_assert_held(&gt->mcr_lock);
254 
255 	if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70)) {
256 		/*
257 		 * Always leave the hardware in multicast mode when doing reads
258 		 * (see comment about Wa_22013088509 below) and only change it
259 		 * to unicast mode when doing writes of a specific instance.
260 		 *
261 		 * No need to save old steering reg value.
262 		 */
263 		intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR,
264 				      REG_FIELD_PREP(MTL_MCR_GROUPID, group) |
265 				      REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance) |
266 				      (rw_flag == FW_REG_READ ? GEN11_MCR_MULTICAST : 0));
267 	} else if (GRAPHICS_VER(uncore->i915) >= 11) {
268 		mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
269 		mcr_ss = GEN11_MCR_SLICE(group) | GEN11_MCR_SUBSLICE(instance);
270 
271 		/*
272 		 * Wa_22013088509
273 		 *
274 		 * The setting of the multicast/unicast bit usually wouldn't
275 		 * matter for read operations (which always return the value
276 		 * from a single register instance regardless of how that bit
277 		 * is set), but some platforms have a workaround requiring us
278 		 * to remain in multicast mode for reads.  There's no real
279 		 * downside to this, so we'll just go ahead and do so on all
280 		 * platforms; we'll only clear the multicast bit from the mask
281 		 * when exlicitly doing a write operation.
282 		 */
283 		if (rw_flag == FW_REG_WRITE)
284 			mcr_mask |= GEN11_MCR_MULTICAST;
285 
286 		mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
287 		old_mcr = mcr;
288 
289 		mcr &= ~mcr_mask;
290 		mcr |= mcr_ss;
291 		intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
292 	} else {
293 		mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
294 		mcr_ss = GEN8_MCR_SLICE(group) | GEN8_MCR_SUBSLICE(instance);
295 
296 		mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
297 		old_mcr = mcr;
298 
299 		mcr &= ~mcr_mask;
300 		mcr |= mcr_ss;
301 		intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
302 	}
303 
304 	if (rw_flag == FW_REG_READ)
305 		val = intel_uncore_read_fw(uncore, mcr_reg_cast(reg));
306 	else
307 		intel_uncore_write_fw(uncore, mcr_reg_cast(reg), value);
308 
309 	/*
310 	 * For pre-MTL platforms, we need to restore the old value of the
311 	 * steering control register to ensure that implicit steering continues
312 	 * to behave as expected.  For MTL and beyond, we need only reinstate
313 	 * the 'multicast' bit (and only if we did a write that cleared it).
314 	 */
315 	if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70) && rw_flag == FW_REG_WRITE)
316 		intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
317 	else if (GRAPHICS_VER_FULL(uncore->i915) < IP_VER(12, 70))
318 		intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, old_mcr);
319 
320 	return val;
321 }
322 
rw_with_mcr_steering(struct intel_gt * gt,i915_mcr_reg_t reg,u8 rw_flag,int group,int instance,u32 value)323 static u32 rw_with_mcr_steering(struct intel_gt *gt,
324 				i915_mcr_reg_t reg, u8 rw_flag,
325 				int group, int instance,
326 				u32 value)
327 {
328 	struct intel_uncore *uncore = gt->uncore;
329 	enum forcewake_domains fw_domains;
330 	unsigned long flags;
331 	u32 val;
332 
333 	fw_domains = intel_uncore_forcewake_for_reg(uncore, mcr_reg_cast(reg),
334 						    rw_flag);
335 	fw_domains |= intel_uncore_forcewake_for_reg(uncore,
336 						     GEN8_MCR_SELECTOR,
337 						     FW_REG_READ | FW_REG_WRITE);
338 
339 	intel_gt_mcr_lock(gt, &flags);
340 	spin_lock(&uncore->lock);
341 	intel_uncore_forcewake_get__locked(uncore, fw_domains);
342 
343 	val = rw_with_mcr_steering_fw(gt, reg, rw_flag, group, instance, value);
344 
345 	intel_uncore_forcewake_put__locked(uncore, fw_domains);
346 	spin_unlock(&uncore->lock);
347 	intel_gt_mcr_unlock(gt, flags);
348 
349 	return val;
350 }
351 
352 /**
353  * intel_gt_mcr_lock - Acquire MCR steering lock
354  * @gt: GT structure
355  * @flags: storage to save IRQ flags to
356  *
357  * Performs locking to protect the steering for the duration of an MCR
358  * operation.  On MTL and beyond, a hardware lock will also be taken to
359  * serialize access not only for the driver, but also for external hardware and
360  * firmware agents.
361  *
362  * Context: Takes gt->mcr_lock.  uncore->lock should *not* be held when this
363  *          function is called, although it may be acquired after this
364  *          function call.
365  */
intel_gt_mcr_lock(struct intel_gt * gt,unsigned long * flags)366 void intel_gt_mcr_lock(struct intel_gt *gt, unsigned long *flags)
367 {
368 	unsigned long __flags;
369 	int err = 0;
370 
371 	lockdep_assert_not_held(&gt->uncore->lock);
372 
373 	/*
374 	 * Starting with MTL, we need to coordinate not only with other
375 	 * driver threads, but also with hardware/firmware agents.  A dedicated
376 	 * locking register is used.
377 	 */
378 	if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
379 		err = wait_for(intel_uncore_read_fw(gt->uncore,
380 						    MTL_STEER_SEMAPHORE) == 0x1, 100);
381 
382 	/*
383 	 * Even on platforms with a hardware lock, we'll continue to grab
384 	 * a software spinlock too for lockdep purposes.  If the hardware lock
385 	 * was already acquired, there should never be contention on the
386 	 * software lock.
387 	 */
388 	spin_lock_irqsave(&gt->mcr_lock, __flags);
389 
390 	*flags = __flags;
391 
392 	/*
393 	 * In theory we should never fail to acquire the HW semaphore; this
394 	 * would indicate some hardware/firmware is misbehaving and not
395 	 * releasing it properly.
396 	 */
397 	if (err == -ETIMEDOUT) {
398 		gt_err_ratelimited(gt, "hardware MCR steering semaphore timed out");
399 		add_taint_for_CI(gt->i915, TAINT_WARN);  /* CI is now unreliable */
400 	}
401 }
402 
403 /**
404  * intel_gt_mcr_unlock - Release MCR steering lock
405  * @gt: GT structure
406  * @flags: IRQ flags to restore
407  *
408  * Releases the lock acquired by intel_gt_mcr_lock().
409  *
410  * Context: Releases gt->mcr_lock
411  */
intel_gt_mcr_unlock(struct intel_gt * gt,unsigned long flags)412 void intel_gt_mcr_unlock(struct intel_gt *gt, unsigned long flags)
413 {
414 	spin_unlock_irqrestore(&gt->mcr_lock, flags);
415 
416 	if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
417 		intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1);
418 }
419 
420 /**
421  * intel_gt_mcr_read - read a specific instance of an MCR register
422  * @gt: GT structure
423  * @reg: the MCR register to read
424  * @group: the MCR group
425  * @instance: the MCR instance
426  *
427  * Context: Takes and releases gt->mcr_lock
428  *
429  * Returns the value read from an MCR register after steering toward a specific
430  * group/instance.
431  */
intel_gt_mcr_read(struct intel_gt * gt,i915_mcr_reg_t reg,int group,int instance)432 u32 intel_gt_mcr_read(struct intel_gt *gt,
433 		      i915_mcr_reg_t reg,
434 		      int group, int instance)
435 {
436 	return rw_with_mcr_steering(gt, reg, FW_REG_READ, group, instance, 0);
437 }
438 
439 /**
440  * intel_gt_mcr_unicast_write - write a specific instance of an MCR register
441  * @gt: GT structure
442  * @reg: the MCR register to write
443  * @value: value to write
444  * @group: the MCR group
445  * @instance: the MCR instance
446  *
447  * Write an MCR register in unicast mode after steering toward a specific
448  * group/instance.
449  *
450  * Context: Calls a function that takes and releases gt->mcr_lock
451  */
intel_gt_mcr_unicast_write(struct intel_gt * gt,i915_mcr_reg_t reg,u32 value,int group,int instance)452 void intel_gt_mcr_unicast_write(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value,
453 				int group, int instance)
454 {
455 	rw_with_mcr_steering(gt, reg, FW_REG_WRITE, group, instance, value);
456 }
457 
458 /**
459  * intel_gt_mcr_multicast_write - write a value to all instances of an MCR register
460  * @gt: GT structure
461  * @reg: the MCR register to write
462  * @value: value to write
463  *
464  * Write an MCR register in multicast mode to update all instances.
465  *
466  * Context: Takes and releases gt->mcr_lock
467  */
intel_gt_mcr_multicast_write(struct intel_gt * gt,i915_mcr_reg_t reg,u32 value)468 void intel_gt_mcr_multicast_write(struct intel_gt *gt,
469 				  i915_mcr_reg_t reg, u32 value)
470 {
471 	unsigned long flags;
472 
473 	intel_gt_mcr_lock(gt, &flags);
474 
475 	/*
476 	 * Ensure we have multicast behavior, just in case some non-i915 agent
477 	 * left the hardware in unicast mode.
478 	 */
479 	if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
480 		intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
481 
482 	intel_uncore_write(gt->uncore, mcr_reg_cast(reg), value);
483 
484 	intel_gt_mcr_unlock(gt, flags);
485 }
486 
487 /**
488  * intel_gt_mcr_multicast_write_fw - write a value to all instances of an MCR register
489  * @gt: GT structure
490  * @reg: the MCR register to write
491  * @value: value to write
492  *
493  * Write an MCR register in multicast mode to update all instances.  This
494  * function assumes the caller is already holding any necessary forcewake
495  * domains; use intel_gt_mcr_multicast_write() in cases where forcewake should
496  * be obtained automatically.
497  *
498  * Context: The caller must hold gt->mcr_lock.
499  */
intel_gt_mcr_multicast_write_fw(struct intel_gt * gt,i915_mcr_reg_t reg,u32 value)500 void intel_gt_mcr_multicast_write_fw(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value)
501 {
502 	lockdep_assert_held(&gt->mcr_lock);
503 
504 	/*
505 	 * Ensure we have multicast behavior, just in case some non-i915 agent
506 	 * left the hardware in unicast mode.
507 	 */
508 	if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
509 		intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
510 
511 	intel_uncore_write_fw(gt->uncore, mcr_reg_cast(reg), value);
512 }
513 
514 /**
515  * intel_gt_mcr_multicast_rmw - Performs a multicast RMW operations
516  * @gt: GT structure
517  * @reg: the MCR register to read and write
518  * @clear: bits to clear during RMW
519  * @set: bits to set during RMW
520  *
521  * Performs a read-modify-write on an MCR register in a multicast manner.
522  * This operation only makes sense on MCR registers where all instances are
523  * expected to have the same value.  The read will target any non-terminated
524  * instance and the write will be applied to all instances.
525  *
526  * This function assumes the caller is already holding any necessary forcewake
527  * domains; use intel_gt_mcr_multicast_rmw() in cases where forcewake should
528  * be obtained automatically.
529  *
530  * Context: Calls functions that take and release gt->mcr_lock
531  *
532  * Returns the old (unmodified) value read.
533  */
intel_gt_mcr_multicast_rmw(struct intel_gt * gt,i915_mcr_reg_t reg,u32 clear,u32 set)534 u32 intel_gt_mcr_multicast_rmw(struct intel_gt *gt, i915_mcr_reg_t reg,
535 			       u32 clear, u32 set)
536 {
537 	u32 val = intel_gt_mcr_read_any(gt, reg);
538 
539 	intel_gt_mcr_multicast_write(gt, reg, (val & ~clear) | set);
540 
541 	return val;
542 }
543 
544 /*
545  * reg_needs_read_steering - determine whether a register read requires
546  *     explicit steering
547  * @gt: GT structure
548  * @reg: the register to check steering requirements for
549  * @type: type of multicast steering to check
550  *
551  * Determines whether @reg needs explicit steering of a specific type for
552  * reads.
553  *
554  * Returns false if @reg does not belong to a register range of the given
555  * steering type, or if the default (subslice-based) steering IDs are suitable
556  * for @type steering too.
557  */
reg_needs_read_steering(struct intel_gt * gt,i915_mcr_reg_t reg,enum intel_steering_type type)558 static bool reg_needs_read_steering(struct intel_gt *gt,
559 				    i915_mcr_reg_t reg,
560 				    enum intel_steering_type type)
561 {
562 	u32 offset = i915_mmio_reg_offset(reg);
563 	const struct intel_mmio_range *entry;
564 
565 	if (likely(!gt->steering_table[type]))
566 		return false;
567 
568 	if (IS_GSI_REG(offset))
569 		offset += gt->uncore->gsi_offset;
570 
571 	for (entry = gt->steering_table[type]; entry->end; entry++) {
572 		if (offset >= entry->start && offset <= entry->end)
573 			return true;
574 	}
575 
576 	return false;
577 }
578 
579 /*
580  * get_nonterminated_steering - determines valid IDs for a class of MCR steering
581  * @gt: GT structure
582  * @type: multicast register type
583  * @group: Group ID returned
584  * @instance: Instance ID returned
585  *
586  * Determines group and instance values that will steer reads of the specified
587  * MCR class to a non-terminated instance.
588  */
get_nonterminated_steering(struct intel_gt * gt,enum intel_steering_type type,u8 * group,u8 * instance)589 static void get_nonterminated_steering(struct intel_gt *gt,
590 				       enum intel_steering_type type,
591 				       u8 *group, u8 *instance)
592 {
593 	u32 dss;
594 
595 	switch (type) {
596 	case L3BANK:
597 		*group = 0;		/* unused */
598 		*instance = __ffs(gt->info.l3bank_mask);
599 		break;
600 	case MSLICE:
601 		GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));
602 		*group = __ffs(gt->info.mslice_mask);
603 		*instance = 0;	/* unused */
604 		break;
605 	case LNCF:
606 		/*
607 		 * An LNCF is always present if its mslice is present, so we
608 		 * can safely just steer to LNCF 0 in all cases.
609 		 */
610 		GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));
611 		*group = __ffs(gt->info.mslice_mask) << 1;
612 		*instance = 0;	/* unused */
613 		break;
614 	case GAM:
615 		*group = IS_DG2(gt->i915) ? 1 : 0;
616 		*instance = 0;
617 		break;
618 	case DSS:
619 		dss = intel_sseu_find_first_xehp_dss(&gt->info.sseu, 0, 0);
620 		*group = dss / GEN_DSS_PER_GSLICE;
621 		*instance = dss % GEN_DSS_PER_GSLICE;
622 		break;
623 	case INSTANCE0:
624 		/*
625 		 * There are a lot of MCR types for which instance (0, 0)
626 		 * will always provide a non-terminated value.
627 		 */
628 		*group = 0;
629 		*instance = 0;
630 		break;
631 	case OADDRM:
632 		if ((VDBOX_MASK(gt) | VEBOX_MASK(gt) | gt->info.sfc_mask) & BIT(0))
633 			*group = 0;
634 		else
635 			*group = 1;
636 		*instance = 0;
637 		break;
638 	default:
639 		MISSING_CASE(type);
640 		*group = 0;
641 		*instance = 0;
642 	}
643 }
644 
645 /**
646  * intel_gt_mcr_get_nonterminated_steering - find group/instance values that
647  *    will steer a register to a non-terminated instance
648  * @gt: GT structure
649  * @reg: register for which the steering is required
650  * @group: return variable for group steering
651  * @instance: return variable for instance steering
652  *
653  * This function returns a group/instance pair that is guaranteed to work for
654  * read steering of the given register. Note that a value will be returned even
655  * if the register is not replicated and therefore does not actually require
656  * steering.
657  */
intel_gt_mcr_get_nonterminated_steering(struct intel_gt * gt,i915_mcr_reg_t reg,u8 * group,u8 * instance)658 void intel_gt_mcr_get_nonterminated_steering(struct intel_gt *gt,
659 					     i915_mcr_reg_t reg,
660 					     u8 *group, u8 *instance)
661 {
662 	int type;
663 
664 	for (type = 0; type < NUM_STEERING_TYPES; type++) {
665 		if (reg_needs_read_steering(gt, reg, type)) {
666 			get_nonterminated_steering(gt, type, group, instance);
667 			return;
668 		}
669 	}
670 
671 	*group = gt->default_steering.groupid;
672 	*instance = gt->default_steering.instanceid;
673 }
674 
675 /**
676  * intel_gt_mcr_read_any_fw - reads one instance of an MCR register
677  * @gt: GT structure
678  * @reg: register to read
679  *
680  * Reads a GT MCR register.  The read will be steered to a non-terminated
681  * instance (i.e., one that isn't fused off or powered down by power gating).
682  * This function assumes the caller is already holding any necessary forcewake
683  * domains; use intel_gt_mcr_read_any() in cases where forcewake should be
684  * obtained automatically.
685  *
686  * Context: The caller must hold gt->mcr_lock.
687  *
688  * Returns the value from a non-terminated instance of @reg.
689  */
intel_gt_mcr_read_any_fw(struct intel_gt * gt,i915_mcr_reg_t reg)690 u32 intel_gt_mcr_read_any_fw(struct intel_gt *gt, i915_mcr_reg_t reg)
691 {
692 	int type;
693 	u8 group, instance;
694 
695 	lockdep_assert_held(&gt->mcr_lock);
696 
697 	for (type = 0; type < NUM_STEERING_TYPES; type++) {
698 		if (reg_needs_read_steering(gt, reg, type)) {
699 			get_nonterminated_steering(gt, type, &group, &instance);
700 			return rw_with_mcr_steering_fw(gt, reg,
701 						       FW_REG_READ,
702 						       group, instance, 0);
703 		}
704 	}
705 
706 	return intel_uncore_read_fw(gt->uncore, mcr_reg_cast(reg));
707 }
708 
709 /**
710  * intel_gt_mcr_read_any - reads one instance of an MCR register
711  * @gt: GT structure
712  * @reg: register to read
713  *
714  * Reads a GT MCR register.  The read will be steered to a non-terminated
715  * instance (i.e., one that isn't fused off or powered down by power gating).
716  *
717  * Context: Calls a function that takes and releases gt->mcr_lock.
718  *
719  * Returns the value from a non-terminated instance of @reg.
720  */
intel_gt_mcr_read_any(struct intel_gt * gt,i915_mcr_reg_t reg)721 u32 intel_gt_mcr_read_any(struct intel_gt *gt, i915_mcr_reg_t reg)
722 {
723 	int type;
724 	u8 group, instance;
725 
726 	for (type = 0; type < NUM_STEERING_TYPES; type++) {
727 		if (reg_needs_read_steering(gt, reg, type)) {
728 			get_nonterminated_steering(gt, type, &group, &instance);
729 			return rw_with_mcr_steering(gt, reg,
730 						    FW_REG_READ,
731 						    group, instance, 0);
732 		}
733 	}
734 
735 	return intel_uncore_read(gt->uncore, mcr_reg_cast(reg));
736 }
737 
report_steering_type(struct drm_printer * p,struct intel_gt * gt,enum intel_steering_type type,bool dump_table)738 static void report_steering_type(struct drm_printer *p,
739 				 struct intel_gt *gt,
740 				 enum intel_steering_type type,
741 				 bool dump_table)
742 {
743 	const struct intel_mmio_range *entry;
744 	u8 group, instance;
745 
746 	BUILD_BUG_ON(ARRAY_SIZE(intel_steering_types) != NUM_STEERING_TYPES);
747 
748 	if (!gt->steering_table[type]) {
749 		drm_printf(p, "%s steering: uses default steering\n",
750 			   intel_steering_types[type]);
751 		return;
752 	}
753 
754 	get_nonterminated_steering(gt, type, &group, &instance);
755 	drm_printf(p, "%s steering: group=0x%x, instance=0x%x\n",
756 		   intel_steering_types[type], group, instance);
757 
758 	if (!dump_table)
759 		return;
760 
761 	for (entry = gt->steering_table[type]; entry->end; entry++)
762 		drm_printf(p, "\t0x%06x - 0x%06x\n", entry->start, entry->end);
763 }
764 
intel_gt_mcr_report_steering(struct drm_printer * p,struct intel_gt * gt,bool dump_table)765 void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt,
766 				  bool dump_table)
767 {
768 	/*
769 	 * Starting with MTL we no longer have default steering;
770 	 * all ranges are explicitly steered.
771 	 */
772 	if (GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 70))
773 		drm_printf(p, "Default steering: group=0x%x, instance=0x%x\n",
774 			   gt->default_steering.groupid,
775 			   gt->default_steering.instanceid);
776 
777 	if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
778 		for (int i = 0; i < NUM_STEERING_TYPES; i++)
779 			if (gt->steering_table[i])
780 				report_steering_type(p, gt, i, dump_table);
781 	} else if (IS_PONTEVECCHIO(gt->i915)) {
782 		report_steering_type(p, gt, INSTANCE0, dump_table);
783 	} else if (HAS_MSLICE_STEERING(gt->i915)) {
784 		report_steering_type(p, gt, MSLICE, dump_table);
785 		report_steering_type(p, gt, LNCF, dump_table);
786 	}
787 }
788 
789 /**
790  * intel_gt_mcr_get_ss_steering - returns the group/instance steering for a SS
791  * @gt: GT structure
792  * @dss: DSS ID to obtain steering for
793  * @group: pointer to storage for steering group ID
794  * @instance: pointer to storage for steering instance ID
795  *
796  * Returns the steering IDs (via the @group and @instance parameters) that
797  * correspond to a specific subslice/DSS ID.
798  */
intel_gt_mcr_get_ss_steering(struct intel_gt * gt,unsigned int dss,unsigned int * group,unsigned int * instance)799 void intel_gt_mcr_get_ss_steering(struct intel_gt *gt, unsigned int dss,
800 				   unsigned int *group, unsigned int *instance)
801 {
802 	if (IS_PONTEVECCHIO(gt->i915)) {
803 		*group = dss / GEN_DSS_PER_CSLICE;
804 		*instance = dss % GEN_DSS_PER_CSLICE;
805 	} else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) {
806 		*group = dss / GEN_DSS_PER_GSLICE;
807 		*instance = dss % GEN_DSS_PER_GSLICE;
808 	} else {
809 		*group = dss / GEN_MAX_SS_PER_HSW_SLICE;
810 		*instance = dss % GEN_MAX_SS_PER_HSW_SLICE;
811 		return;
812 	}
813 }
814 
815 /**
816  * intel_gt_mcr_wait_for_reg - wait until MCR register matches expected state
817  * @gt: GT structure
818  * @reg: the register to read
819  * @mask: mask to apply to register value
820  * @value: value to wait for
821  * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
822  * @slow_timeout_ms: slow timeout in millisecond
823  *
824  * This routine waits until the target register @reg contains the expected
825  * @value after applying the @mask, i.e. it waits until ::
826  *
827  *     (intel_gt_mcr_read_any_fw(gt, reg) & mask) == value
828  *
829  * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
830  * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
831  * must be not larger than 20,0000 microseconds.
832  *
833  * This function is basically an MCR-friendly version of
834  * __intel_wait_for_register_fw().  Generally this function will only be used
835  * on GAM registers which are a bit special --- although they're MCR registers,
836  * reads (e.g., waiting for status updates) are always directed to the primary
837  * instance.
838  *
839  * Note that this routine assumes the caller holds forcewake asserted, it is
840  * not suitable for very long waits.
841  *
842  * Context: Calls a function that takes and releases gt->mcr_lock
843  * Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
844  */
intel_gt_mcr_wait_for_reg(struct intel_gt * gt,i915_mcr_reg_t reg,u32 mask,u32 value,unsigned int fast_timeout_us,unsigned int slow_timeout_ms)845 int intel_gt_mcr_wait_for_reg(struct intel_gt *gt,
846 			      i915_mcr_reg_t reg,
847 			      u32 mask,
848 			      u32 value,
849 			      unsigned int fast_timeout_us,
850 			      unsigned int slow_timeout_ms)
851 {
852 	int ret;
853 
854 	lockdep_assert_not_held(&gt->mcr_lock);
855 
856 #define done ((intel_gt_mcr_read_any(gt, reg) & mask) == value)
857 
858 	/* Catch any overuse of this function */
859 	might_sleep_if(slow_timeout_ms);
860 	GEM_BUG_ON(fast_timeout_us > 20000);
861 	GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms);
862 
863 	ret = -ETIMEDOUT;
864 	if (fast_timeout_us && fast_timeout_us <= 20000)
865 		ret = _wait_for_atomic(done, fast_timeout_us, 0);
866 	if (ret && slow_timeout_ms)
867 		ret = wait_for(done, slow_timeout_ms);
868 
869 	return ret;
870 #undef done
871 }
872