diff options
| author | Greg Kroah-Hartman <gregkh@linuxfoundation.org> | 2026-09-11 11:51:26 +0200 |
|---|---|---|
| committer | Greg Kroah-Hartman <gregkh@linuxfoundation.org> | 2026-09-11 11:51:26 +0200 |
| commit | ffb45b46184f54bf84d95e82df46932294b2031a (patch) | |
| tree | cc199e0ec572fb2bf65fa4eb144f0e562bb34687 /mm/damon | |
| parent | 5eccd39d8efa3bc8d557be50f202bbf023837eed (diff) | |
| parent | a300e35c0a4b4a38fb53742ea6e2a203c98ee523 (diff) | |
| download | linux-rolling-stable.tar.gz linux-rolling-stable.zip | |
Merge v7.2.5linux-rolling-stable
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Diffstat (limited to 'mm/damon')
| -rw-r--r-- | mm/damon/core.c | 27 | ||||
| -rw-r--r-- | mm/damon/ops-common.c | 5 | ||||
| -rw-r--r-- | mm/damon/paddr.c | 20 | ||||
| -rw-r--r-- | mm/damon/sysfs-schemes.c | 18 | ||||
| -rw-r--r-- | mm/damon/sysfs.c | 29 | ||||
| -rw-r--r-- | mm/damon/tests/core-kunit.h | 33 | ||||
| -rw-r--r-- | mm/damon/tests/vaddr-kunit.h | 5 | ||||
| -rw-r--r-- | mm/damon/vaddr.c | 34 |
8 files changed, 103 insertions, 68 deletions
diff --git a/mm/damon/core.c b/mm/damon/core.c index cff932b3317d..a86812d457c1 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -579,6 +579,7 @@ struct damos *damon_new_scheme(struct damos_access_pattern *pattern, INIT_LIST_HEAD(&scheme->ops_filters); scheme->stat = (struct damos_stat){}; scheme->max_nr_snapshots = 0; + scheme->last_applied = NULL; INIT_LIST_HEAD(&scheme->list); scheme->quota = *(damos_quota_init(quota)); @@ -1670,7 +1671,7 @@ static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) return sz; } -static void damon_split_region_at(struct damon_target *t, +static int damon_split_region_at(struct damon_target *t, struct damon_region *r, unsigned long sz_r); /* @@ -1696,11 +1697,13 @@ static unsigned long damon_apply_min_nr_regions(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region_safe(r, next, t) { while (damon_sz_region(r) > max_region_sz) { - damon_split_region_at(t, r, max_region_sz); + if (damon_split_region_at(t, r, max_region_sz)) + goto out; r = damon_next_region(r); } } } +out: return max_region_sz; } @@ -3163,15 +3166,20 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, max_thres = c->attrs.aggr_interval / (c->attrs.sample_interval ? c->attrs.sample_interval : 1); - do { + while (true) { nr_regions = 0; damon_for_each_target(t, c) { damon_merge_regions_of(t, threshold, sz_limit); nr_regions += damon_nr_regions(t); } - threshold = max(1, threshold * 2); - } while (nr_regions > c->attrs.max_nr_regions && - threshold / 2 < max_thres); + if (nr_regions <= c->attrs.max_nr_regions || + max_thres <= threshold) + break; + if (threshold < max_thres / 2) + threshold = max(1, threshold * 2); + else + threshold = max_thres; + } } #ifdef CONFIG_DAMON_DEBUG_SANITY @@ -3194,8 +3202,10 @@ static void damon_verify_split_region_at(struct damon_region *r, * * r the region to be split * sz_r size of the first sub-region that will be made + * + * Return: 0 on success, negative error code otherwise. */ -static void damon_split_region_at(struct damon_target *t, +static int damon_split_region_at(struct damon_target *t, struct damon_region *r, unsigned long sz_r) { struct damon_region *new; @@ -3203,7 +3213,7 @@ static void damon_split_region_at(struct damon_target *t, damon_verify_split_region_at(r, sz_r); new = damon_new_region(r->ar.start + sz_r, r->ar.end); if (!new) - return; + return -ENOMEM; r->ar.end = new->ar.start; @@ -3215,6 +3225,7 @@ static void damon_split_region_at(struct damon_target *t, memcpy(new->probe_hits, r->probe_hits, sizeof(r->probe_hits)); damon_insert_region(new, r, damon_next_region(r), t); + return 0; } /* Split every region in the given target into 'nr_subs' regions */ diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 0bcad6b1e5b9..6a969b1d2987 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -111,8 +111,9 @@ int damon_hot_score(struct damon_ctx *c, struct damon_region *r, unsigned int age_weight = s->quota.weight_age; int hotness; - freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / - damon_max_nr_accesses(&c->attrs); + freq_subscore = mult_frac(r->nr_accesses_bp / 10000, + DAMON_MAX_SUBSCORE, + damon_max_nr_accesses(&c->attrs)); age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000; if (age_in_sec) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index f7613ce279a2..b3507b1f6249 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -65,7 +65,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) } } -static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) +static bool damon_pa_young(phys_addr_t paddr) { struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); bool accessed; @@ -74,7 +74,6 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) return false; accessed = damon_folio_young(folio); - *folio_sz = folio_size(folio); folio_put(folio); return accessed; } @@ -82,23 +81,12 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) static void __damon_pa_check_access(struct damon_region *r, struct damon_attrs *attrs, unsigned long addr_unit) { - static phys_addr_t last_addr; - static unsigned long last_folio_sz = PAGE_SIZE; - static bool last_accessed; + bool accessed; phys_addr_t sampling_addr = damon_pa_phys_addr( r->sampling_addr, addr_unit); - /* If the region is in the last checked page, reuse the result */ - if (ALIGN_DOWN(last_addr, last_folio_sz) == - ALIGN_DOWN(sampling_addr, last_folio_sz)) { - damon_update_region_access_rate(r, last_accessed, attrs); - return; - } - - last_accessed = damon_pa_young(sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); - - last_addr = sampling_addr; + accessed = damon_pa_young(sampling_addr); + damon_update_region_access_rate(r, accessed, attrs); } static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 3cbeccd436e4..e2c8716be6c9 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -332,6 +332,7 @@ static void damon_sysfs_scheme_regions_rm_dirs( list_for_each_entry_safe(r, next, ®ions->regions_list, list) { damos_sysfs_region_rm_dirs(r); list_del(&r->list); + kobject_del(&r->kobj); kobject_put(&r->kobj); regions->nr_regions--; } @@ -911,8 +912,10 @@ static void damon_sysfs_scheme_filters_rm_dirs( struct damon_sysfs_scheme_filter **filters_arr = filters->filters_arr; int i; - for (i = 0; i < filters->nr; i++) + for (i = 0; i < filters->nr; i++) { + kobject_del(&filters_arr[i]->kobj); kobject_put(&filters_arr[i]->kobj); + } filters->nr = 0; kfree(filters_arr); filters->filters_arr = NULL; @@ -1460,8 +1463,10 @@ static void damos_sysfs_quota_goals_rm_dirs( struct damos_sysfs_quota_goal **goals_arr = goals->goals_arr; int i; - for (i = 0; i < goals->nr; i++) + for (i = 0; i < goals->nr; i++) { + kobject_del(&goals_arr[i]->kobj); kobject_put(&goals_arr[i]->kobj); + } goals->nr = 0; kfree(goals_arr); goals->goals_arr = NULL; @@ -2138,8 +2143,10 @@ static void damos_sysfs_dests_rm_dirs( struct damos_sysfs_dest **dests_arr = dests->dests_arr; int i; - for (i = 0; i < dests->nr; i++) + for (i = 0; i < dests->nr; i++) { + kobject_del(&dests_arr[i]->kobj); kobject_put(&dests_arr[i]->kobj); + } dests->nr = 0; kfree(dests_arr); dests->dests_arr = NULL; @@ -2681,6 +2688,7 @@ void damon_sysfs_schemes_rm_dirs(struct damon_sysfs_schemes *schemes) for (i = 0; i < schemes->nr; i++) { damon_sysfs_scheme_rm_dirs(schemes_arr[i]); + kobject_del(&schemes_arr[i]->kobj); kobject_put(&schemes_arr[i]->kobj); } schemes->nr = 0; @@ -2722,13 +2730,15 @@ static int damon_sysfs_schemes_add_dirs(struct damon_sysfs_schemes *schemes, goto out; err = damon_sysfs_scheme_add_dirs(scheme); if (err) - goto out; + goto del_out; schemes_arr[i] = scheme; schemes->nr++; } return 0; +del_out: + kobject_del(&scheme->kobj); out: damon_sysfs_schemes_rm_dirs(schemes); kobject_put(&scheme->kobj); diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index a9e187158067..26cdad603ad0 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -105,8 +105,10 @@ static void damon_sysfs_regions_rm_dirs(struct damon_sysfs_regions *regions) struct damon_sysfs_region **regions_arr = regions->regions_arr; int i; - for (i = 0; i < regions->nr; i++) + for (i = 0; i < regions->nr; i++) { + kobject_del(®ions_arr[i]->kobj); kobject_put(®ions_arr[i]->kobj); + } regions->nr = 0; kfree(regions_arr); regions->regions_arr = NULL; @@ -331,6 +333,7 @@ static void damon_sysfs_targets_rm_dirs(struct damon_sysfs_targets *targets) for (i = 0; i < targets->nr; i++) { damon_sysfs_target_rm_dirs(targets_arr[i]); + kobject_del(&targets_arr[i]->kobj); kobject_put(&targets_arr[i]->kobj); } targets->nr = 0; @@ -369,13 +372,15 @@ static int damon_sysfs_targets_add_dirs(struct damon_sysfs_targets *targets, err = damon_sysfs_target_add_dirs(target); if (err) - goto out; + goto del_out; targets_arr[i] = target; targets->nr++; } return 0; +del_out: + kobject_del(&target->kobj); out: damon_sysfs_targets_rm_dirs(targets); kobject_put(&target->kobj); @@ -1640,6 +1645,7 @@ static void damon_sysfs_contexts_rm_dirs(struct damon_sysfs_contexts *contexts) for (i = 0; i < contexts->nr; i++) { damon_sysfs_context_rm_dirs(contexts_arr[i]); + kobject_del(&contexts_arr[i]->kobj); kobject_put(&contexts_arr[i]->kobj); } contexts->nr = 0; @@ -1678,13 +1684,15 @@ static int damon_sysfs_contexts_add_dirs(struct damon_sysfs_contexts *contexts, err = damon_sysfs_context_add_dirs(context); if (err) - goto out; + goto del_out; contexts_arr[i] = context; contexts->nr++; } return 0; +del_out: + kobject_del(&context->kobj); out: damon_sysfs_contexts_rm_dirs(contexts); kobject_put(&context->kobj); @@ -2042,16 +2050,18 @@ static inline bool damon_sysfs_kdamond_running( static int damon_sysfs_apply_inputs(struct damon_ctx *ctx, struct damon_sysfs_context *sys_ctx) { + enum damon_ops_id ops_id; int err; - err = damon_select_ops(ctx, sys_ctx->ops_id); + ops_id = READ_ONCE(sys_ctx->ops_id); + err = damon_select_ops(ctx, ops_id); if (err) return err; - ctx->addr_unit = sys_ctx->addr_unit; + ctx->addr_unit = READ_ONCE(sys_ctx->addr_unit); /* addr_unit is respected by only DAMON_OPS_PADDR */ - if (sys_ctx->ops_id == DAMON_OPS_PADDR) + if (ops_id == DAMON_OPS_PADDR) ctx->min_region_sz = max( - DAMON_MIN_REGION_SZ / sys_ctx->addr_unit, 1); + DAMON_MIN_REGION_SZ / ctx->addr_unit, 1); ctx->pause = sys_ctx->pause; err = damon_sysfs_set_attrs(ctx, sys_ctx->attrs); if (err) @@ -2499,6 +2509,7 @@ static void damon_sysfs_kdamonds_rm_dirs(struct damon_sysfs_kdamonds *kdamonds) for (i = 0; i < kdamonds->nr; i++) { damon_sysfs_kdamond_rm_dirs(kdamonds_arr[i]); + kobject_del(&kdamonds_arr[i]->kobj); kobject_put(&kdamonds_arr[i]->kobj); } kdamonds->nr = 0; @@ -2553,13 +2564,15 @@ static int damon_sysfs_kdamonds_add_dirs(struct damon_sysfs_kdamonds *kdamonds, err = damon_sysfs_kdamond_add_dirs(kdamond); if (err) - goto out; + goto del_out; kdamonds_arr[i] = kdamond; kdamonds->nr++; } return 0; +del_out: + kobject_del(&kdamond->kobj); out: damon_sysfs_kdamonds_rm_dirs(kdamonds); kobject_put(&kdamond->kobj); diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index fcf7c7fadb5f..fbcc882dccc9 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -157,6 +157,10 @@ static void damon_test_split_at(struct kunit *test) r->age = 10; damon_add_region(r, t); damon_split_region_at(t, r, 25); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; + KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); KUNIT_EXPECT_EQ(test, r->ar.end, 25ul); @@ -169,6 +173,7 @@ static void damon_test_split_at(struct kunit *test) KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses); KUNIT_EXPECT_EQ(test, r->age, r_new->age); +out: damon_free_target(t); } @@ -259,11 +264,14 @@ static void damon_test_merge_regions_of(struct kunit *test) damon_merge_regions_of(t, 9, 9999); /* 0-112, 114-130, 130-156, 156-170, 170-230, 230-10170 */ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 6u); + if (damon_nr_regions(t) != 6) + goto out; for (i = 0; i < 6; i++) { r = __nth_region_of(t, i); KUNIT_EXPECT_EQ(test, r->ar.start, saddrs[i]); KUNIT_EXPECT_EQ(test, r->ar.end, eaddrs[i]); } +out: damon_free_target(t); } @@ -725,6 +733,7 @@ static void damos_test_commit_quota_goals_for(struct kunit *test, struct damos_quota_goal *goal, *next; bool skip = true; int i; + int nr_dst = 0, nr_src = 0; INIT_LIST_HEAD(&dst.goals); INIT_LIST_HEAD(&src.goals); @@ -747,6 +756,14 @@ static void damos_test_commit_quota_goals_for(struct kunit *test, damos_commit_quota_goals(&dst, &src); + damos_for_each_quota_goal(goal, &dst) + nr_dst++; + damos_for_each_quota_goal(goal, &src) + nr_src++; + KUNIT_EXPECT_EQ(test, nr_dst, nr_src); + if (nr_dst != nr_src) + goto out; + i = 0; damos_for_each_quota_goal(goal, (&dst)) { KUNIT_EXPECT_EQ(test, goal->metric, src_goals[i].metric); @@ -894,6 +911,8 @@ static void damos_test_commit_dests_for(struct kunit *test, skip = false; KUNIT_EXPECT_EQ(test, dst.nr_dests, src_nr_dests); + if (dst.nr_dests != src_nr_dests) + goto out; for (i = 0; i < dst.nr_dests; i++) { KUNIT_EXPECT_EQ(test, dst.node_id_arr[i], src_node_id_arr[i]); KUNIT_EXPECT_EQ(test, dst.weight_arr[i], src_weight_arr[i]); @@ -1152,14 +1171,19 @@ static void damon_test_commit_target_regions_for(struct kunit *test, kunit_skip(test, "src target setup fail"); } damon_commit_target_regions(dst_target, src_target, 1); + + KUNIT_EXPECT_EQ(test, damon_nr_regions(dst_target), nr_expect_regions); + if (damon_nr_regions(dst_target) != nr_expect_regions) + goto out; + i = 0; damon_for_each_region(r, dst_target) { KUNIT_EXPECT_EQ(test, r->ar.start, expect_start_end[i][0]); KUNIT_EXPECT_EQ(test, r->ar.end, expect_start_end[i][1]); i++; } - KUNIT_EXPECT_EQ(test, damon_nr_regions(dst_target), nr_expect_regions); - KUNIT_EXPECT_EQ(test, i, nr_expect_regions); + +out: damon_free_target(dst_target); damon_free_target(src_target); } @@ -1251,6 +1275,8 @@ static void damos_test_filter_out(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 1); KUNIT_EXPECT_EQ(test, r->ar.end, 2); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; r2 = damon_next_region(r); KUNIT_EXPECT_EQ(test, r2->ar.start, 2); KUNIT_EXPECT_EQ(test, r2->ar.end, 4); @@ -1265,11 +1291,14 @@ static void damos_test_filter_out(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 2); KUNIT_EXPECT_EQ(test, r->ar.end, 6); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; r2 = damon_next_region(r); KUNIT_EXPECT_EQ(test, r2->ar.start, 6); KUNIT_EXPECT_EQ(test, r2->ar.end, 8); damon_destroy_region(r2, t); +out: damon_free_target(t); damos_free_filter(f); } diff --git a/mm/damon/tests/vaddr-kunit.h b/mm/damon/tests/vaddr-kunit.h index 61f844336ffb..6a95441d193a 100644 --- a/mm/damon/tests/vaddr-kunit.h +++ b/mm/damon/tests/vaddr-kunit.h @@ -158,12 +158,17 @@ static void damon_do_test_apply_three_regions(struct kunit *test, kunit_skip(test, "second damon_set_regions() fail"); } + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), nr_expected / 2); + if (damon_nr_regions(t) != nr_expected / 2) + goto out; + for (i = 0; i < nr_expected / 2; i++) { r = __nth_region_of(t, i); KUNIT_EXPECT_EQ(test, r->ar.start, expected[i * 2]); KUNIT_EXPECT_EQ(test, r->ar.end, expected[i * 2 + 1]); } +out: damon_destroy_target(t, NULL); } diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 2c1c1952c008..404785128ffd 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -382,8 +382,6 @@ static void damon_va_prepare_access_checks(struct damon_ctx *ctx) } struct damon_young_walk_private { - /* size of the folio for the access checked virtual memory address */ - unsigned long *folio_sz; bool young; }; @@ -410,7 +408,6 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = HPAGE_PMD_SIZE; huge_out: spin_unlock(ptl); return 0; @@ -429,7 +426,6 @@ huge_out: if (pte_young(ptent) || !folio_test_idle(folio) || mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = folio_size(folio); out: pte_unmap_unlock(pte, ptl); return 0; @@ -457,7 +453,6 @@ static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask, if (pte_young(entry) || !folio_test_idle(folio) || mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = huge_page_size(h); folio_put(folio); @@ -469,11 +464,9 @@ out: #define damon_young_hugetlb_entry NULL #endif /* CONFIG_HUGETLB_PAGE */ -static bool damon_va_young(struct mm_struct *mm, unsigned long addr, - unsigned long *folio_sz) +static bool damon_va_young(struct mm_struct *mm, unsigned long addr) { struct damon_young_walk_private arg = { - .folio_sz = folio_sz, .young = false, }; @@ -493,29 +486,18 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr, * r the region to be checked */ static void __damon_va_check_access(struct mm_struct *mm, - struct damon_region *r, bool same_target, + struct damon_region *r, struct damon_attrs *attrs) { - static unsigned long last_addr; - static unsigned long last_folio_sz = PAGE_SIZE; - static bool last_accessed; + bool accessed; if (!mm) { damon_update_region_access_rate(r, false, attrs); return; } - /* If the region is in the last checked page, reuse the result */ - if (same_target && (ALIGN_DOWN(last_addr, last_folio_sz) == - ALIGN_DOWN(r->sampling_addr, last_folio_sz))) { - damon_update_region_access_rate(r, last_accessed, attrs); - return; - } - - last_accessed = damon_va_young(mm, r->sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); - - last_addr = r->sampling_addr; + accessed = damon_va_young(mm, r->sampling_addr); + damon_update_region_access_rate(r, accessed, attrs); } static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) @@ -524,16 +506,12 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) struct mm_struct *mm; struct damon_region *r; unsigned int max_nr_accesses = 0; - bool same_target; damon_for_each_target(t, ctx) { mm = damon_get_mm(t); - same_target = false; damon_for_each_region(r, t) { - __damon_va_check_access(mm, r, same_target, - &ctx->attrs); + __damon_va_check_access(mm, r, &ctx->attrs); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); - same_target = true; } if (mm) mmput(mm); |
