Lines Matching refs:sds
6720 struct sched_domain_shared *sds; in set_idle_cores() local
6722 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); in set_idle_cores()
6723 if (sds) in set_idle_cores()
6724 WRITE_ONCE(sds->has_idle_cores, val); in set_idle_cores()
6729 struct sched_domain_shared *sds; in test_idle_cores() local
6731 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); in test_idle_cores()
6732 if (sds) in test_idle_cores()
6733 return READ_ONCE(sds->has_idle_cores); in test_idle_cores()
8980 static inline void init_sd_lb_stats(struct sd_lb_stats *sds) in init_sd_lb_stats() argument
8989 *sds = (struct sd_lb_stats){ in init_sd_lb_stats()
9263 static bool asym_smt_can_pull_tasks(int dst_cpu, struct sd_lb_stats *sds, in asym_smt_can_pull_tasks() argument
9271 local_is_smt = sds->local->flags & SD_SHARE_CPUCAPACITY; in asym_smt_can_pull_tasks()
9297 int local_busy_cpus = sds->local->group_weight - in asym_smt_can_pull_tasks()
9298 sds->local_stat.idle_cpus; in asym_smt_can_pull_tasks()
9312 if (!sds->local_stat.sum_nr_running) in asym_smt_can_pull_tasks()
9323 sched_asym(struct lb_env *env, struct sd_lb_stats *sds, struct sg_lb_stats *sgs, in sched_asym() argument
9327 if ((sds->local->flags & SD_SHARE_CPUCAPACITY) || in sched_asym()
9329 return asym_smt_can_pull_tasks(env->dst_cpu, sds, sgs, group); in sched_asym()
9356 struct sd_lb_stats *sds, in update_sg_lb_stats() argument
9365 local_group = group == sds->local; in update_sg_lb_stats()
9422 sched_asym(env, sds, sgs, group)) { in update_sg_lb_stats()
9448 struct sd_lb_stats *sds, in update_sd_pick_busiest() argument
9452 struct sg_lb_stats *busiest = &sds->busiest_stat; in update_sd_pick_busiest()
9467 sds->local_stat.group_type != group_has_spare)) in update_sd_pick_busiest()
9497 if (sched_asym_prefer(sg->asym_prefer_cpu, sds->busiest->asym_prefer_cpu)) in update_sd_pick_busiest()
9995 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) in update_sd_lb_stats() argument
9999 struct sg_lb_stats *local = &sds->local_stat; in update_sd_lb_stats()
10010 sds->local = sg; in update_sd_lb_stats()
10018 update_sg_lb_stats(env, sds, sg, sgs, &sg_status); in update_sd_lb_stats()
10024 if (update_sd_pick_busiest(env, sds, sg, sgs)) { in update_sd_lb_stats()
10025 sds->busiest = sg; in update_sd_lb_stats()
10026 sds->busiest_stat = *sgs; in update_sd_lb_stats()
10031 sds->total_load += sgs->group_load; in update_sd_lb_stats()
10032 sds->total_capacity += sgs->group_capacity; in update_sd_lb_stats()
10039 sds->prefer_sibling = child && child->flags & SD_PREFER_SIBLING; in update_sd_lb_stats()
10043 env->fbq_type = fbq_classify_group(&sds->busiest_stat); in update_sd_lb_stats()
10070 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) in calculate_imbalance() argument
10074 local = &sds->local_stat; in calculate_imbalance()
10075 busiest = &sds->busiest_stat; in calculate_imbalance()
10149 if (busiest->group_weight == 1 || sds->prefer_sibling) { in calculate_imbalance()
10206 sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / in calculate_imbalance()
10207 sds->total_capacity; in calculate_imbalance()
10220 (busiest->avg_load - sds->avg_load) * busiest->group_capacity, in calculate_imbalance()
10221 (sds->avg_load - local->avg_load) * local->group_capacity in calculate_imbalance()
10260 struct sd_lb_stats sds; in find_busiest_group() local
10262 init_sd_lb_stats(&sds); in find_busiest_group()
10268 update_sd_lb_stats(env, &sds); in find_busiest_group()
10271 if (!sds.busiest) in find_busiest_group()
10274 busiest = &sds.busiest_stat; in find_busiest_group()
10299 local = &sds.local_stat; in find_busiest_group()
10320 sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) / in find_busiest_group()
10321 sds.total_capacity; in find_busiest_group()
10327 if (local->avg_load >= sds.avg_load) in find_busiest_group()
10340 if (sds.prefer_sibling && local->group_type == group_has_spare && in find_busiest_group()
10375 calculate_imbalance(env, &sds); in find_busiest_group()
10376 return env->imbalance ? sds.busiest : NULL; in find_busiest_group()
11216 struct sched_domain_shared *sds; in nohz_balancer_kick() local
11300 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); in nohz_balancer_kick()
11301 if (sds) { in nohz_balancer_kick()
11311 nr_busy = atomic_read(&sds->nr_busy_cpus); in nohz_balancer_kick()