diff options
| -rw-r--r-- | arch/x86/kernel/itmt.c | 8 | ||||
| -rw-r--r-- | kernel/sched/core.c | 10 | ||||
| -rw-r--r-- | kernel/sched/deadline.c | 4 | ||||
| -rw-r--r-- | kernel/sched/fair.c | 60 | ||||
| -rw-r--r-- | kernel/sched/rt.c | 4 |
5 files changed, 64 insertions, 22 deletions
diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c index 243a769fdd97..85ebde361d6a 100644 --- a/arch/x86/kernel/itmt.c +++ b/arch/x86/kernel/itmt.c @@ -110,18 +110,14 @@ int sched_set_itmt_support(void) arch_debugfs_dir, &sysctl_sched_itmt_enabled, &dfs_sched_itmt_fops); - if (IS_ERR_OR_NULL(dfs_sched_itmt)) { + if (IS_ERR(dfs_sched_itmt)) dfs_sched_itmt = NULL; - return -ENOMEM; - } dfs_sched_core_prio = debugfs_create_file("sched_core_priority", 0644, arch_debugfs_dir, NULL, &sched_core_priority_fops); - if (IS_ERR_OR_NULL(dfs_sched_core_prio)) { + if (IS_ERR(dfs_sched_core_prio)) dfs_sched_core_prio = NULL; - return -ENOMEM; - } sched_itmt_capable = true; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 449ccd871be8..b998ef6b87af 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3742,11 +3742,17 @@ static inline void ttwu_do_wakeup(struct task_struct *p) void update_rq_avg_idle(struct rq *rq) { - u64 delta = rq_clock(rq) - rq->idle_stamp; - u64 max = 2*rq->max_idle_balance_cost; + u64 idle_stamp = rq->idle_stamp; + u64 delta, max; + + if (!idle_stamp) + return; + + delta = rq_clock(rq) - idle_stamp; update_avg(&rq->avg_idle, delta); + max = 2 * rq->max_idle_balance_cost; if (rq->avg_idle > max) rq->avg_idle = max; rq->idle_stamp = 0; diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 857dbe3519a8..0663c00c41c0 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -3028,8 +3028,8 @@ static struct task_struct *pick_next_pushable_dl_task(struct rq *rq) next_node = rb_first_cached(&rq->dl.pushable_dl_tasks_root); while (next_node) { i = __node_2_pdl(next_node); - /* make sure task isn't on_cpu (possible with proxy-exec) */ - if (!task_on_cpu(rq, i)) { + /* skip tasks that cannot be migrated */ + if (!task_on_cpu(rq, i) && !is_migration_disabled(i)) { p = i; break; } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 8dff37059faf..ade1eceb39b8 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6978,14 +6978,14 @@ static int tg_throttle_down(struct task_group *tg, void *data) static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) { struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); - struct sched_entity *curr = cfs_rq->curr; + struct sched_entity *curr = cfs_rq->h_curr; struct rq *rq = rq_of(cfs_rq); scoped_guard(raw_spinlock, &cfs_b->lock) { u64 target_runtime = 1; /* - * If cfs_rq->curr is still runnable, we are here from an + * If cfs_rq->h_curr is still runnable, we are here from an * update_curr(). Request sysctl_sched_cfs_bandwidth_slice * worth of bandwidth to continue running. * @@ -7192,7 +7192,7 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) if (!list_empty(&cfs_rq->throttled_csd_list)) continue; - if (cfs_rq->curr) { + if (cfs_rq->h_curr) { update_rq_clock(rq); update_curr(cfs_rq); } @@ -10057,7 +10057,7 @@ again: /* Might not have done put_prev_entity() */ if (cfs_rq->curr && cfs_rq->curr->on_rq) - update_curr(cfs_rq); + update_curr_eevdf(cfs_rq); se = pick_next_entity(rq, true); if (!se) @@ -10160,7 +10160,7 @@ static void yield_task_fair(struct rq *rq) /* * Update run-time statistics of the 'current'. */ - update_curr(cfs_rq); + update_curr_eevdf(cfs_rq); /* * Tell update_rq_clock() that we've just updated, * so we don't do microscopic update in schedule() @@ -10691,17 +10691,40 @@ static enum llc_mig can_migrate_llc(int src_cpu, int dst_cpu, return mig_llc; } +static inline bool task_misfits_asym_cpu(struct lb_env *env, struct task_struct *p) +{ + /* + * On asymmetric CPU capacity domains, do not let cache-aware + * balancing pull the task onto a destination CPU that cannot + * accommodate it. Doing so would turn the task into a misfit on + * the destination, trading a cache-locality gain for a capacity + * loss. If the task already does not fit its source CPU, the move + * cannot make things worse, so let the LLC preference decide. + */ + if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && p && + !task_fits_cpu(p, env->dst_cpu) && + task_fits_cpu(p, env->src_cpu)) + return true; + + return false; +} + /* * Check if task p can migrate from source LLC to * destination LLC in terms of cache aware load balance. */ -static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, +static enum llc_mig can_migrate_llc_task(struct lb_env *env, struct task_struct *p) { struct mm_struct *mm; bool to_pref; - int cpu; + int cpu, src_cpu, dst_cpu; + + if (task_misfits_asym_cpu(env, p)) + return mig_forbid; + src_cpu = env->src_cpu; + dst_cpu = env->dst_cpu; mm = p->mm; if (!mm) return mig_unrestricted; @@ -10758,6 +10781,14 @@ alb_break_llc(struct lb_env *env) unsigned long util = 0; struct task_struct *cur; + /* + * Migrating misfit tasks from current CPU + * to CPU with a better fit. + * Prioritize that over LLC preference. + */ + if (env->migration_type == migrate_misfit) + return false; + if (env->src_rq->nr_running <= 1) return true; @@ -10765,7 +10796,8 @@ alb_break_llc(struct lb_env *env) if (cur && cur->sched_class == &fair_sched_class) util = task_util(cur); - if (can_migrate_llc(env->src_cpu, env->dst_cpu, + if (task_misfits_asym_cpu(env, cur) || + can_migrate_llc(env->src_cpu, env->dst_cpu, util, false) == mig_forbid) return true; } @@ -10805,8 +10837,7 @@ static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env) READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu)) return true; - if (can_migrate_llc_task(env->src_cpu, - env->dst_cpu, p) != mig_forbid) + if (can_migrate_llc_task(env, p) != mig_forbid) return false; return true; @@ -11870,6 +11901,15 @@ static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs, return false; /* + * On asymmetric domains, group_misfit_task_load + * should be prioritized to move tasks to CPU that fit them + * over aggregating tasks to their preferred LLC. + */ + if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && + sgs->group_misfit_task_load) + return false; + + /* * Skip cache aware tagging if nr_balanced_failed is sufficiently high. * Threshold of cache_nice_tries is set to 1 higher than nr_balance_failed * to avoid excessive task migration at the same time. diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index e6e5f8a2caaf..85303add726d 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1872,8 +1872,8 @@ static struct task_struct *pick_next_pushable_task(struct rq *rq) return NULL; plist_for_each_entry(i, head, pushable_tasks) { - /* make sure task isn't on_cpu (possible with proxy-exec) */ - if (!task_on_cpu(rq, i)) { + /* skip tasks that cannot be migrated */ + if (!task_on_cpu(rq, i) && !is_migration_disabled(i)) { p = i; break; } |