Lines Matching refs:sg_cpu
156 static void sugov_get_util(struct sugov_cpu *sg_cpu) in sugov_get_util() argument
158 struct rq *rq = cpu_rq(sg_cpu->cpu); in sugov_get_util()
160 sg_cpu->bw_dl = cpu_bw_dl(rq); in sugov_get_util()
161 sg_cpu->util = effective_cpu_util(sg_cpu->cpu, cpu_util_cfs(sg_cpu->cpu), in sugov_get_util()
176 static bool sugov_iowait_reset(struct sugov_cpu *sg_cpu, u64 time, in sugov_iowait_reset() argument
179 s64 delta_ns = time - sg_cpu->last_update; in sugov_iowait_reset()
185 sg_cpu->iowait_boost = set_iowait_boost ? IOWAIT_BOOST_MIN : 0; in sugov_iowait_reset()
186 sg_cpu->iowait_boost_pending = set_iowait_boost; in sugov_iowait_reset()
205 static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, in sugov_iowait_boost() argument
211 if (sg_cpu->iowait_boost && in sugov_iowait_boost()
212 sugov_iowait_reset(sg_cpu, time, set_iowait_boost)) in sugov_iowait_boost()
220 if (sg_cpu->iowait_boost_pending) in sugov_iowait_boost()
222 sg_cpu->iowait_boost_pending = true; in sugov_iowait_boost()
225 if (sg_cpu->iowait_boost) { in sugov_iowait_boost()
226 sg_cpu->iowait_boost = in sugov_iowait_boost()
227 min_t(unsigned int, sg_cpu->iowait_boost << 1, SCHED_CAPACITY_SCALE); in sugov_iowait_boost()
232 sg_cpu->iowait_boost = IOWAIT_BOOST_MIN; in sugov_iowait_boost()
253 static void sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time, in sugov_iowait_apply() argument
259 if (!sg_cpu->iowait_boost) in sugov_iowait_apply()
263 if (sugov_iowait_reset(sg_cpu, time, false)) in sugov_iowait_apply()
266 if (!sg_cpu->iowait_boost_pending) { in sugov_iowait_apply()
270 sg_cpu->iowait_boost >>= 1; in sugov_iowait_apply()
271 if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) { in sugov_iowait_apply()
272 sg_cpu->iowait_boost = 0; in sugov_iowait_apply()
277 sg_cpu->iowait_boost_pending = false; in sugov_iowait_apply()
283 boost = (sg_cpu->iowait_boost * max_cap) >> SCHED_CAPACITY_SHIFT; in sugov_iowait_apply()
284 boost = uclamp_rq_util_with(cpu_rq(sg_cpu->cpu), boost, NULL); in sugov_iowait_apply()
285 if (sg_cpu->util < boost) in sugov_iowait_apply()
286 sg_cpu->util = boost; in sugov_iowait_apply()
290 static bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) in sugov_cpu_is_busy() argument
292 unsigned long idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu); in sugov_cpu_is_busy()
293 bool ret = idle_calls == sg_cpu->saved_idle_calls; in sugov_cpu_is_busy()
295 sg_cpu->saved_idle_calls = idle_calls; in sugov_cpu_is_busy()
299 static inline bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) { return false; } in sugov_cpu_is_busy() argument
306 static inline void ignore_dl_rate_limit(struct sugov_cpu *sg_cpu) in ignore_dl_rate_limit() argument
308 if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_dl) in ignore_dl_rate_limit()
309 sg_cpu->sg_policy->limits_changed = true; in ignore_dl_rate_limit()
312 static inline bool sugov_update_single_common(struct sugov_cpu *sg_cpu, in sugov_update_single_common() argument
316 sugov_iowait_boost(sg_cpu, time, flags); in sugov_update_single_common()
317 sg_cpu->last_update = time; in sugov_update_single_common()
319 ignore_dl_rate_limit(sg_cpu); in sugov_update_single_common()
321 if (!sugov_should_update_freq(sg_cpu->sg_policy, time)) in sugov_update_single_common()
324 sugov_get_util(sg_cpu); in sugov_update_single_common()
325 sugov_iowait_apply(sg_cpu, time, max_cap); in sugov_update_single_common()
333 struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); in sugov_update_single_freq() local
334 struct sugov_policy *sg_policy = sg_cpu->sg_policy; in sugov_update_single_freq()
339 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); in sugov_update_single_freq()
341 if (!sugov_update_single_common(sg_cpu, time, max_cap, flags)) in sugov_update_single_freq()
344 next_f = get_next_freq(sg_policy, sg_cpu->util, max_cap); in sugov_update_single_freq()
351 if (!uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)) && in sugov_update_single_freq()
352 sugov_cpu_is_busy(sg_cpu) && next_f < sg_policy->next_freq) { in sugov_update_single_freq()
379 struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); in sugov_update_single_perf() local
380 unsigned long prev_util = sg_cpu->util; in sugov_update_single_perf()
393 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); in sugov_update_single_perf()
395 if (!sugov_update_single_common(sg_cpu, time, max_cap, flags)) in sugov_update_single_perf()
404 if (!uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)) && in sugov_update_single_perf()
405 sugov_cpu_is_busy(sg_cpu) && sg_cpu->util < prev_util) in sugov_update_single_perf()
406 sg_cpu->util = prev_util; in sugov_update_single_perf()
408 cpufreq_driver_adjust_perf(sg_cpu->cpu, map_util_perf(sg_cpu->bw_dl), in sugov_update_single_perf()
409 map_util_perf(sg_cpu->util), max_cap); in sugov_update_single_perf()
411 sg_cpu->sg_policy->last_freq_update_time = time; in sugov_update_single_perf()
414 static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time) in sugov_next_freq_shared() argument
416 struct sugov_policy *sg_policy = sg_cpu->sg_policy; in sugov_next_freq_shared()
421 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); in sugov_next_freq_shared()
438 struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); in sugov_update_shared() local
439 struct sugov_policy *sg_policy = sg_cpu->sg_policy; in sugov_update_shared()
444 sugov_iowait_boost(sg_cpu, time, flags); in sugov_update_shared()
445 sg_cpu->last_update = time; in sugov_update_shared()
447 ignore_dl_rate_limit(sg_cpu); in sugov_update_shared()
450 next_f = sugov_next_freq_shared(sg_cpu, time); in sugov_update_shared()
770 struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu); in sugov_start() local
772 memset(sg_cpu, 0, sizeof(*sg_cpu)); in sugov_start()
773 sg_cpu->cpu = cpu; in sugov_start()
774 sg_cpu->sg_policy = sg_policy; in sugov_start()
785 struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu); in sugov_start() local
787 cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu); in sugov_start()