summaryrefslogtreecommitdiff
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/compaction.c98
-rw-r--r--mm/gup.c10
-rw-r--r--mm/huge_memory.c24
-rw-r--r--mm/hugetlb.c38
-rw-r--r--mm/hugetlb_vmemmap.c23
-rw-r--r--mm/internal.h10
-rw-r--r--mm/kasan/quarantine.c7
-rw-r--r--mm/kmemleak.c77
-rw-r--r--mm/madvise.c4
-rw-r--r--mm/memcontrol-v1.c6
-rw-r--r--mm/memcontrol.c13
-rw-r--r--mm/mempolicy.c29
-rw-r--r--mm/migrate.c8
-rw-r--r--mm/migrate_device.c13
-rw-r--r--mm/mm_init.c10
-rw-r--r--mm/page_alloc.c76
-rw-r--r--mm/page_owner.c10
-rw-r--r--mm/page_vma_mapped.c8
-rw-r--r--mm/pagewalk.c6
-rw-r--r--mm/rmap.c31
-rw-r--r--mm/slub.c78
-rw-r--r--mm/sparse-vmemmap.c4
-rw-r--r--mm/swapfile.c15
-rw-r--r--mm/vmalloc.c58
-rw-r--r--mm/vmscan.c40
-rw-r--r--mm/zsmalloc.c11
-rw-r--r--mm/zswap.c13
27 files changed, 457 insertions, 263 deletions
diff --git a/mm/compaction.c b/mm/compaction.c
index f08765ade014..65fd08aabb2c 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -1381,12 +1381,44 @@ static bool suitable_migration_source(struct compact_control *cc,
if (pageblock_skip_persistent(page))
return false;
- if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction)
+ /*
+ * Background compaction produces blocks for the zone at
+ * large, with no particular allocation context. Allow all
+ * block types, including CMA.
+ */
+ if (!cc->direct_compaction)
return true;
block_mt = get_pageblock_migratetype(page);
- if (cc->migratetype == MIGRATE_MOVABLE)
+ /*
+ * CMA pages can only be taken by ALLOC_CMA requests. For anybody
+ * else, vacating a CMA block consumes free pages the caller
+ * could have used, and produces free pages it cannot.
+ */
+ if (is_migrate_cma(block_mt) && !(cc->alloc_flags & ALLOC_CMA))
+ return false;
+
+ /*
+ * Per default, scans are restricted to blocks compatible with
+ * the request, to prevent cross-contamination. Once
+ * compaction priority escalates to synchronous scans, though,
+ * scan all blocks to try to make forward progress. For
+ * movable request, this likely helps little: there shouldn't
+ * be many migratable pages inside non-movable blocks besides
+ * allocator fallbacks. For non-movable requests, this helps a
+ * lot, as they can finally scan movable blocks.
+ */
+ if (cc->mode != MIGRATE_ASYNC)
+ return true;
+
+ /*
+ * Prevent <pageblock_order unmovable/reclaimable requests from
+ * polluting movable blocks through fallbacks. Whole-block production
+ * (directly requested, or defrag_mode) is exempt as the allocator
+ * claims and converts these.
+ */
+ if (cc->migratetype == MIGRATE_MOVABLE || cc->order >= pageblock_order)
return is_migrate_movable(block_mt);
else
return block_mt == cc->migratetype;
@@ -1974,12 +2006,12 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc)
return pfn;
/*
- * Only allow kcompactd and direct requests for movable pages to
- * quickly clear out a MOVABLE pageblock for allocation. This
- * reduces the risk that a large movable pageblock is freed for
- * an unmovable/reclaimable small allocation.
+ * Prevent <pageblock_order unmovable/reclaimable requests from
+ * polluting movable blocks through fallbacks. Whole-block production
+ * is exempt as the allocator claims and converts these.
*/
- if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE)
+ if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE &&
+ cc->order < pageblock_order)
return pfn;
/*
@@ -2770,9 +2802,8 @@ out:
static enum compact_result compact_zone_order(struct zone *zone, int order,
gfp_t gfp_mask, enum compact_priority prio,
unsigned int alloc_flags, int highest_zoneidx,
- struct page **capture)
+ struct capture_control *capc)
{
- enum compact_result ret;
struct compact_control cc = {
.order = order,
.search_order = order,
@@ -2787,54 +2818,24 @@ static enum compact_result compact_zone_order(struct zone *zone, int order,
.ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
.ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
};
- struct capture_control capc = {
- .cc = &cc,
- .page = NULL,
- };
- /*
- * Make sure the structs are really initialized before we expose the
- * capture control, in case we are interrupted and the interrupt handler
- * frees a page.
- */
- barrier();
- WRITE_ONCE(current->capture_control, &capc);
-
- ret = compact_zone(&cc, &capc);
-
- /*
- * Make sure we hide capture control first before we read the captured
- * page pointer, otherwise an interrupt could free and capture a page
- * and we would leak it.
- */
- WRITE_ONCE(current->capture_control, NULL);
- *capture = READ_ONCE(capc.page);
- /*
- * Technically, it is also possible that compaction is skipped but
- * the page is still captured out of luck(IRQ came and freed the page).
- * Returning COMPACT_SUCCESS in such cases helps in properly accounting
- * the COMPACT[STALL|FAIL] when compaction is skipped.
- */
- if (*capture)
- ret = COMPACT_SUCCESS;
-
- return ret;
+ return compact_zone(&cc, capc);
}
/**
* try_to_compact_pages - Direct compact to satisfy a high-order allocation
* @gfp_mask: The GFP mask of the current allocation
- * @order: The order of the current allocation
+ * @order: The order to try to make available
* @alloc_flags: The allocation flags of the current allocation
* @ac: The context of current allocation
* @prio: Determines how hard direct compaction should try to succeed
- * @capture: Pointer to free page created by compaction will be stored here
+ * @capc: Free page capture bypassing the freelist
*
* This is the main entry point for direct page compaction.
*/
enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
unsigned int alloc_flags, const struct alloc_context *ac,
- enum compact_priority prio, struct page **capture)
+ enum compact_priority prio, struct capture_control *capc)
{
struct zoneref *z;
struct zone *zone;
@@ -2861,8 +2862,17 @@ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
continue;
}
+ WRITE_ONCE(capc->zone, zone);
+
status = compact_zone_order(zone, order, gfp_mask, prio,
- alloc_flags, ac->highest_zoneidx, capture);
+ alloc_flags, ac->highest_zoneidx, capc);
+
+ WRITE_ONCE(capc->zone, NULL);
+
+ /* Stop if a page has been captured */
+ if (READ_ONCE(capc->page))
+ status = COMPACT_SUCCESS;
+
rc = max(status, rc);
/* The allocation should succeed, stop compacting */
diff --git a/mm/gup.c b/mm/gup.c
index 0692119b7904..bde05664fe9a 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -2273,6 +2273,7 @@ static unsigned long collect_longterm_unpinnable_folios(
for (folio = pofs_get_folio(pofs, i); folio;
folio = pofs_next_folio(folio, pofs, &i)) {
+ const int pin_refs = folio_has_pincount(folio) ? 1 : GUP_PIN_COUNTING_BIAS;
if (folio_is_longterm_pinnable(folio))
continue;
@@ -2287,15 +2288,20 @@ static unsigned long collect_longterm_unpinnable_folios(
continue;
}
+ /*
+ * We drain not only to make the folio_isolate_lru() succeed,
+ * but also to remove any other folio references from LRU
+ * caches.
+ */
if (drained == 0 && folio_may_be_lru_cached(folio) &&
folio_ref_count(folio) !=
- folio_expected_ref_count(folio) + 1) {
+ folio_expected_ref_count(folio) + pin_refs) {
lru_add_drain();
drained = 1;
}
if (drained == 1 && folio_may_be_lru_cached(folio) &&
folio_ref_count(folio) !=
- folio_expected_ref_count(folio) + 1) {
+ folio_expected_ref_count(folio) + pin_refs) {
lru_add_drain_all();
drained = 2;
}
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 5d94fa4c74fd..2860fb43ff57 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2340,8 +2340,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
goto out;
if (unlikely(!pmd_present(orig_pmd))) {
- VM_BUG_ON(thp_migration_supported() &&
- !pmd_is_migration_entry(orig_pmd));
+ VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) &&
+ !pmd_is_device_private_entry(orig_pmd));
goto out;
}
@@ -4027,34 +4027,42 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
XA_STATE(xas, &folio->mapping->i_pages, folio->index);
struct folio *end_folio = folio_next(folio);
bool is_anon = folio_test_anon(folio);
+ struct mem_cgroup *memcg, *old_memcg;
struct address_space *mapping = NULL;
struct anon_vma *anon_vma = NULL;
int old_order = folio_order(folio);
struct folio *new_folio, *next;
int nr_shmem_dropped = 0;
enum ttu_flags ttu_flags = 0;
- int ret;
pgoff_t end = 0;
+ int ret;
VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio);
if (folio != page_folio(split_at) || folio != page_folio(lock_at)) {
ret = -EINVAL;
- goto out;
+ goto out_no_memcg;
}
if (new_order >= old_order) {
ret = -EINVAL;
- goto out;
+ goto out_no_memcg;
}
ret = folio_check_splittable(folio, new_order, split_type);
if (ret) {
VM_WARN_ONCE(ret == -EINVAL, "Tried to split an unsplittable folio");
- goto out;
+ goto out_no_memcg;
}
+ /*
+ * switch to folio's memcg as xarray node allocation can happen and
+ * needs to charge to it.
+ */
+ memcg = get_mem_cgroup_from_folio(folio);
+ old_memcg = set_active_memcg(memcg);
+
if (is_anon) {
/*
* The caller does not necessarily hold an mmap_lock that would
@@ -4197,6 +4205,10 @@ out_unlock:
if (mapping)
i_mmap_unlock_read(mapping);
out:
+ /* restore to caller's old_memcg */
+ set_active_memcg(old_memcg);
+ mem_cgroup_put(memcg);
+out_no_memcg:
xas_destroy(&xas);
if (is_pmd_order(old_order))
count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index e93c4d2456aa..5c9c6a842c41 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3353,7 +3353,7 @@ static void __init gather_bootmem_prealloc_parallel(unsigned long start,
gather_bootmem_prealloc_node(nid);
}
-static void __init gather_bootmem_prealloc(void)
+void __init hugetlb_bootmem_struct_page_init(void)
{
struct padata_mt_job job = {
.thread_fn = gather_bootmem_prealloc_parallel,
@@ -3365,6 +3365,31 @@ static void __init gather_bootmem_prealloc(void)
.max_threads = num_node_state(N_MEMORY),
.numa_aware = true,
};
+#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
+ struct zone *zone;
+
+ for_each_zone(zone) {
+ for (int i = 0; i < NR_VMEMMAP_TAILS; i++) {
+ struct page *tail, *p;
+ unsigned int order;
+
+ tail = zone->vmemmap_tails[i];
+ if (!tail)
+ continue;
+
+ order = i + VMEMMAP_TAIL_MIN_ORDER;
+ p = page_to_virt(tail);
+ /*
+ * prep_and_add_bootmem_folios() can access pageblock
+ * flags on bootmem HugeTLB pages, so initialize the
+ * shared tail struct pages here before bootmem folios
+ * start using them.
+ */
+ for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++)
+ init_compound_tail(p + j, NULL, order, zone);
+ }
+ }
+#endif
padata_do_multithreaded(&job);
}
@@ -3557,7 +3582,7 @@ static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h)
* - For gigantic pages, this is called early in the boot process and
* pages are allocated from memblock allocated or something similar.
* Gigantic pages are actually added to pools later with the routine
- * gather_bootmem_prealloc.
+ * hugetlb_bootmem_struct_page_init.
* - For non-gigantic pages, this is called later in the boot process after
* all of mm is up and functional. Pages are allocated from buddy and
* then added to hugetlb pools.
@@ -4127,7 +4152,6 @@ static int __init hugetlb_init(void)
}
hugetlb_init_hstates();
- gather_bootmem_prealloc();
report_hugepages();
hugetlb_sysfs_init();
@@ -5157,6 +5181,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
bool adjust_reservation;
unsigned long last_addr_mask;
+ i_mmap_assert_write_locked(vma->vm_file->f_mapping);
WARN_ON(!is_vm_hugetlb_page(vma));
BUG_ON(start & ~huge_page_mask(h));
BUG_ON(end & ~huge_page_mask(h));
@@ -5248,7 +5273,10 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
/*
* Restore the reservation for anonymous page, otherwise the
- * backing page could be stolen by someone.
+ * backing page could be stolen by someone. Restore only on the
+ * last unmap, otherwise the owner could empty its resv map
+ * while the folio is still mapped by a child. Note that holding
+ * i_mmap_lock_write is needed to check the number of mappings.
* If there we are freeing a surplus, do not set the restore
* reservation bit.
*/
@@ -5256,7 +5284,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
spin_lock_irq(&hugetlb_lock);
if (!h->surplus_huge_pages && __vma_private_lock(vma) &&
- folio_test_anon(folio)) {
+ !folio_mapped(folio) && folio_test_anon(folio)) {
folio_set_hugetlb_restore_reserve(folio);
/* Reservation to be adjusted after the spin lock */
adjust_reservation = true;
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 133b46dfb09f..ea6af85bfec1 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -635,12 +635,12 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
* mirrored tail page structs RO.
*/
spfn = (unsigned long)&folio->page;
- epfn = spfn + pages_per_huge_page(h);
+ epfn = spfn + hugetlb_vmemmap_size(h);
vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio),
HUGETLB_VMEMMAP_RESERVE_SIZE);
- register_page_bootmem_memmap(pfn_to_section_nr(spfn),
+ register_page_bootmem_memmap(pfn_to_section_nr(folio_pfn(folio)),
&folio->page,
- HUGETLB_VMEMMAP_RESERVE_SIZE);
+ HUGETLB_VMEMMAP_RESERVE_PAGES);
continue;
}
@@ -870,27 +870,10 @@ static const struct ctl_table hugetlb_vmemmap_sysctls[] = {
static int __init hugetlb_vmemmap_init(void)
{
const struct hstate *h;
- struct zone *zone;
/* HUGETLB_VMEMMAP_RESERVE_SIZE should cover all used struct pages */
BUILD_BUG_ON(__NR_USED_SUBPAGE > HUGETLB_VMEMMAP_RESERVE_PAGES);
- for_each_zone(zone) {
- for (int i = 0; i < NR_VMEMMAP_TAILS; i++) {
- struct page *tail, *p;
- unsigned int order;
-
- tail = zone->vmemmap_tails[i];
- if (!tail)
- continue;
-
- order = i + VMEMMAP_TAIL_MIN_ORDER;
- p = page_to_virt(tail);
- for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++)
- init_compound_tail(p + j, NULL, order, zone);
- }
- }
-
for_each_hstate(h) {
if (hugetlb_vmemmap_optimizable(h)) {
register_sysctl_init("vm", hugetlb_vmemmap_sysctls);
diff --git a/mm/internal.h b/mm/internal.h
index 181e79f1d6a2..5acba3470659 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1059,7 +1059,15 @@ struct compact_control {
* immediately when one is created during the free path.
*/
struct capture_control {
- struct compact_control *cc;
+ struct zone *zone;
+ int migratetype;
+ /*
+ * Allocation request order. May differ from the compaction
+ * order: defrag_mode promotes sub-block allocations to
+ * pageblock-order compaction; capture still matches at the
+ * original allocation order so prep_new_page() is consistent.
+ */
+ int order;
struct page *page;
};
diff --git a/mm/kasan/quarantine.c b/mm/kasan/quarantine.c
index 6958aa713c67..16f4e67beee8 100644
--- a/mm/kasan/quarantine.c
+++ b/mm/kasan/quarantine.c
@@ -355,7 +355,12 @@ void kasan_quarantine_remove_cache(struct kmem_cache *cache)
*/
on_each_cpu(per_cpu_remove_cache, cache, 1);
- for_each_online_cpu(cpu) {
+ /*
+ * A CPU can go offline after on_each_cpu() returns, leaving cache
+ * objects on that CPU's shrink list. Scan all possible CPUs to
+ * drain those lists.
+ */
+ for_each_possible_cpu(cpu) {
sq = per_cpu_ptr(&shrink_qlist, cpu);
raw_spin_lock_irqsave(&sq->lock, flags);
qlist_move_cache(&sq->qlist, &to_free, cache);
diff --git a/mm/kmemleak.c b/mm/kmemleak.c
index e196f53f9b46..e3c9dbde014b 100644
--- a/mm/kmemleak.c
+++ b/mm/kmemleak.c
@@ -1525,22 +1525,25 @@ static int scan_should_stop(void)
/*
* Scan a memory block (exclusive range) for valid pointers and add those
- * found to the gray list.
+ * found to the gray list. Return non-zero if the scan was interrupted.
*/
-static void scan_block(void *_start, void *_end,
- struct kmemleak_object *scanned)
+static int scan_block(void *_start, void *_end,
+ struct kmemleak_object *scanned)
{
unsigned long *ptr;
unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
unsigned long *end = _end - (BYTES_PER_POINTER - 1);
unsigned long flags;
+ int stop = 0;
raw_spin_lock_irqsave(&kmemleak_lock, flags);
for (ptr = start; ptr < end; ptr++) {
unsigned long pointer;
- if (scan_should_stop())
+ if (scan_should_stop()) {
+ stop = 1;
break;
+ }
kasan_disable_current();
pointer = *(unsigned long *)kasan_reset_tag((void *)ptr);
@@ -1550,6 +1553,8 @@ static void scan_block(void *_start, void *_end,
pointer_update_refs(scanned, pointer, OBJECT_PERCPU);
}
raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
+
+ return stop;
}
/*
@@ -1564,7 +1569,7 @@ static void scan_large_block(void *start, void *end)
next = min(start + MAX_SCAN_SIZE, end);
scan_block(start, next, NULL);
start = next;
- cond_resched();
+ cond_resched_tasks_rcu_qs();
}
}
#endif
@@ -1599,7 +1604,7 @@ static void scan_object(struct kmemleak_object *object)
scan_block(start, end, object);
raw_spin_unlock_irqrestore(&object->lock, flags);
- cond_resched();
+ cond_resched_tasks_rcu_qs();
raw_spin_lock_irqsave(&object->lock, flags);
if (!(object->flags & OBJECT_ALLOCATED))
break;
@@ -1621,7 +1626,7 @@ static void scan_object(struct kmemleak_object *object)
break;
raw_spin_unlock_irqrestore(&object->lock, flags);
- cond_resched();
+ cond_resched_tasks_rcu_qs();
raw_spin_lock_irqsave(&object->lock, flags);
} while (object->flags & OBJECT_ALLOCATED);
} else {
@@ -1649,7 +1654,7 @@ static void scan_gray_list(void)
*/
object = list_entry(gray_list.next, typeof(*object), gray_list);
while (&object->gray_list != &gray_list) {
- cond_resched();
+ cond_resched_tasks_rcu_qs();
/* may add new objects to the list */
if (!scan_should_stop())
@@ -1684,7 +1689,7 @@ static void kmemleak_cond_resched(struct kmemleak_object *object)
raw_spin_unlock_irq(&kmemleak_lock);
rcu_read_unlock();
- cond_resched();
+ cond_resched_tasks_rcu_qs();
rcu_read_lock();
raw_spin_lock_irq(&kmemleak_lock);
@@ -1697,6 +1702,43 @@ unlock_put:
}
/*
+ * Scan all task kernel stacks, rescheduling between tasks. Each task is looked
+ * up and pinned within its own RCU read-side section, so no lock is held across
+ * the scan and the walk cannot trip the soft lockup watchdog.
+ */
+static void kmemleak_scan_task_stacks(void)
+{
+ struct pid *pid;
+ int nr = 1;
+ int stop = 0;
+
+ do {
+ struct task_struct *p = NULL;
+
+ rcu_read_lock();
+ pid = find_ge_pid(nr, &init_pid_ns);
+ if (pid) {
+ nr = pid_nr(pid) + 1;
+ p = pid_task(pid, PIDTYPE_PID);
+ if (p)
+ get_task_struct(p);
+ }
+ rcu_read_unlock();
+
+ if (p) {
+ void *stack = try_get_task_stack(p);
+
+ if (stack) {
+ stop = scan_block(stack, stack + THREAD_SIZE, NULL);
+ put_task_stack(p);
+ }
+ put_task_struct(p);
+ }
+ cond_resched_tasks_rcu_qs();
+ } while (pid && !stop);
+}
+
+/*
* Print one leak inline. The hex dump is gated on OBJECT_ALLOCATED so it
* does not touch user memory that was freed concurrently; the rest of the
* report (backtrace, comm, pid) is always emitted since the kmemleak_object
@@ -1866,7 +1908,7 @@ static void kmemleak_scan(void)
struct page *page = pfn_to_online_page(pfn);
if (!(pfn & 63))
- cond_resched();
+ cond_resched_tasks_rcu_qs();
if (!page)
continue;
@@ -1885,19 +1927,8 @@ static void kmemleak_scan(void)
/*
* Scanning the task stacks (may introduce false negatives).
*/
- if (kmemleak_stack_scan) {
- struct task_struct *p, *g;
-
- rcu_read_lock();
- for_each_process_thread(g, p) {
- void *stack = try_get_task_stack(p);
- if (stack) {
- scan_block(stack, stack + THREAD_SIZE, NULL);
- put_task_stack(p);
- }
- }
- rcu_read_unlock();
- }
+ if (kmemleak_stack_scan)
+ kmemleak_scan_task_stacks();
/*
* Scan the objects already referenced from the sections scanned
diff --git a/mm/madvise.c b/mm/madvise.c
index 77552b03d318..31ee9ca2b5c1 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -388,8 +388,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
goto huge_unlock;
if (unlikely(!pmd_present(orig_pmd))) {
- VM_BUG_ON(thp_migration_supported() &&
- !pmd_is_migration_entry(orig_pmd));
+ VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) &&
+ !pmd_is_device_private_entry(orig_pmd));
goto huge_unlock;
}
diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c
index 765069211567..a1776cfacd68 100644
--- a/mm/memcontrol-v1.c
+++ b/mm/memcontrol-v1.c
@@ -1181,13 +1181,13 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
event->unregister_event = mem_cgroup_usage_unregister_event;
} else if (!strcmp(name, "memory.oom_control")) {
pr_warn_once("oom_control is deprecated and will be removed. "
- "Please report your usecase to linux-mm-@kvack.org"
+ "Please report your usecase to linux-mm@kvack.org"
" if you depend on this functionality.\n");
event->register_event = mem_cgroup_oom_register_event;
event->unregister_event = mem_cgroup_oom_unregister_event;
} else if (!strcmp(name, "memory.pressure_level")) {
pr_warn_once("pressure_level is deprecated and will be removed. "
- "Please report your usecase to linux-mm-@kvack.org "
+ "Please report your usecase to linux-mm@kvack.org "
"if you depend on this functionality.\n");
event->register_event = vmpressure_register_event;
event->unregister_event = vmpressure_unregister_event;
@@ -2040,7 +2040,7 @@ static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
struct mem_cgroup *memcg = mem_cgroup_from_css(css);
pr_warn_once("oom_control is deprecated and will be removed. "
- "Please report your usecase to linux-mm-@kvack.org if you "
+ "Please report your usecase to linux-mm@kvack.org if you "
"depend on this functionality.\n");
/* cannot set to root cgroup and only 0 and 1 are allowed */
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 6dc4888a90f3..02f5b414f531 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -4180,9 +4180,11 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
page_counter_init(&memcg->swap, &parent->swap, false);
#ifdef CONFIG_MEMCG_V1
memcg->memory.track_failcnt = !memcg_on_dfl;
+ memcg->memsw.track_failcnt = !memcg_on_dfl;
WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable));
page_counter_init(&memcg->kmem, &parent->kmem, false);
page_counter_init(&memcg->tcpmem, &parent->tcpmem, false);
+ memcg->tcpmem.track_failcnt = !memcg_on_dfl;
#endif
} else {
init_memcg_stats();
@@ -4438,6 +4440,7 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
int index = memcg_stats_index(MEMCG_KMEM);
memcg->vmstats->state[index] += kmem;
+ memcg->vmstats->state_local[index] += kmem;
if (parent)
parent->vmstats->state_pending[index] += kmem;
}
@@ -4455,9 +4458,11 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
int index = memcg_stats_index(NR_SLAB_RECLAIMABLE_B);
lstats->state[index] += slab;
+ lstats->state_local[index] += slab;
if (plstats)
plstats->state_pending[index] += slab;
memcg->vmstats->state[index] += slab;
+ memcg->vmstats->state_local[index] += slab;
if (parent)
parent->vmstats->state_pending[index] += slab;
}
@@ -4466,9 +4471,11 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
int index = memcg_stats_index(NR_SLAB_UNRECLAIMABLE_B);
lstats->state[index] += slab;
+ lstats->state_local[index] += slab;
if (plstats)
plstats->state_pending[index] += slab;
memcg->vmstats->state[index] += slab;
+ memcg->vmstats->state_local[index] += slab;
if (parent)
parent->vmstats->state_pending[index] += slab;
}
@@ -4788,6 +4795,9 @@ static ssize_t memory_high_write(struct kernfs_open_file *of,
unsigned long nr_pages = page_counter_read(&memcg->memory);
unsigned long reclaimed;
+ if (high != READ_ONCE(memcg->memory.high))
+ break;
+
if (nr_pages <= high)
break;
@@ -4839,6 +4849,9 @@ static ssize_t memory_max_write(struct kernfs_open_file *of,
for (;;) {
unsigned long nr_pages = page_counter_read(&memcg->memory);
+ if (max != READ_ONCE(memcg->memory.max))
+ break;
+
if (nr_pages <= max)
break;
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 36699fabd3c2..11cf9705975a 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -653,12 +653,14 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk)
{
struct folio *folio;
struct queue_pages *qp = walk->private;
+ pmd_t pmdval = pmdp_get(pmd);
- if (unlikely(pmd_is_migration_entry(*pmd))) {
- qp->nr_failed++;
+ if (unlikely(!pmd_present(pmdval))) {
+ if (pmd_is_migration_entry(pmdval))
+ qp->nr_failed++;
return;
}
- folio = pmd_folio(*pmd);
+ folio = pmd_folio(pmdval);
if (is_huge_zero_folio(folio)) {
walk->action = ACTION_CONTINUE;
return;
@@ -2057,24 +2059,15 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
bool vma_policy_mof(struct vm_area_struct *vma)
{
struct mempolicy *pol;
+ pgoff_t ilx;
+ bool mof;
- if (vma->vm_ops && vma->vm_ops->get_policy) {
- bool ret = false;
- pgoff_t ilx; /* ignored here */
-
- pol = vma->vm_ops->get_policy(vma, vma->vm_start, &ilx);
- if (pol && (pol->flags & MPOL_F_MOF))
- ret = true;
- mpol_cond_put(pol);
-
- return ret;
- }
-
- pol = vma->vm_policy;
+ pol = __get_vma_policy(vma, vma->vm_start, &ilx);
if (!pol)
pol = get_task_policy(current);
-
- return pol->flags & MPOL_F_MOF;
+ mof = pol->flags & MPOL_F_MOF;
+ mpol_cond_put(pol);
+ return mof;
}
bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
diff --git a/mm/migrate.c b/mm/migrate.c
index dd15a84b2a52..35381ce36a8f 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -371,7 +371,11 @@ static bool remove_migration_pte(struct folio *folio,
continue;
}
#endif
- old_pte = ptep_get(pvmw.pte);
+ if (folio_test_hugetlb(folio))
+ old_pte = huge_ptep_get(vma->vm_mm, pvmw.address,
+ pvmw.pte);
+ else
+ old_pte = ptep_get(pvmw.pte);
if (rmap_walk_arg->map_unused_to_zeropage &&
try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx))
continue;
@@ -1830,7 +1834,7 @@ static int migrate_pages_batch(struct list_head *from,
is_thp = folio_test_pmd_mappable(folio);
nr_pages = folio_nr_pages(folio);
- cond_resched();
+ cond_resched_tasks_rcu_qs();
/*
* The rare folio on the deferred split list should
diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index 908d2d4ec43a..162d29b2807a 100644
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -1183,6 +1183,13 @@ static void __migrate_device_pages(unsigned long *src_pfns,
MIGRATE_PFN_COMPOUND);
goto next;
}
+
+ /*
+ * reset nr so that only first after-split folio
+ * is processed below
+ */
+ VM_WARN_ON_ONCE(folio_test_large(folio));
+ nr = 1;
} else if ((src_pfns[i] & MIGRATE_PFN_MIGRATE) &&
(dst_pfns[i] & MIGRATE_PFN_COMPOUND) &&
!(src_pfns[i] & MIGRATE_PFN_COMPOUND)) {
@@ -1222,6 +1229,12 @@ static void __migrate_device_pages(unsigned long *src_pfns,
folio = page_folio(migrate_pfn_to_page(src_pfns[i+j]));
newfolio = page_folio(migrate_pfn_to_page(dst_pfns[i+j]));
+ /*
+ * folio_free_swap() removed the folio from the swap
+ * cache. Refresh the saved mapping before migration.
+ */
+ mapping = folio_mapping(folio);
+
r = folio_migrate_mapping(mapping, newfolio, folio, extra_cnt);
if (r)
src_pfns[i+j] &= ~MIGRATE_PFN_MIGRATE;
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 498d62c4ece3..ab4285081869 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -2214,10 +2214,13 @@ bool __init deferred_grow_zone(struct zone *zone, unsigned int order)
}
/*
- * There were no pages to initialize and free which means the zone's
- * memory map is completely initialized.
+ * The loop only tests spfn before entering an iteration, so on exit it
+ * may point up to a section past the end of the zone. When it does,
+ * the rest of the zone has already been handed to
+ * deferred_init_memmap_chunk() and nothing is left to initialize.
*/
- pgdat->first_deferred_pfn = nr_pages ? spfn : ULONG_MAX;
+ pgdat->first_deferred_pfn =
+ spfn < zone_end_pfn(zone) ? spfn : ULONG_MAX;
pgdat_resize_unlock(pgdat, &flags);
@@ -2330,6 +2333,7 @@ void __init page_alloc_init_late(void)
/* Reinit limits that are based on free pages after the kernel is up */
files_maxfiles_init();
#endif
+ hugetlb_bootmem_struct_page_init();
/* Accounting of total+free memory is stable at this point. */
mem_init_print_info();
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index ee902a468c2f..f92055827ae9 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -721,14 +721,14 @@ static inline struct capture_control *task_capc(struct zone *zone)
return unlikely(capc) &&
!(current->flags & PF_KTHREAD) &&
!capc->page &&
- capc->cc->zone == zone ? capc : NULL;
+ capc->zone == zone ? capc : NULL;
}
static inline bool
compaction_capture(struct capture_control *capc, struct page *page,
int order, int migratetype)
{
- if (!capc || order != capc->cc->order)
+ if (!capc || order != capc->order)
return false;
/* Do not accidentally pollute CMA or isolated regions*/
@@ -744,12 +744,12 @@ compaction_capture(struct capture_control *capc, struct page *page,
* have trouble finding a high-order free page.
*/
if (order < pageblock_order && migratetype == MIGRATE_MOVABLE &&
- capc->cc->migratetype != MIGRATE_MOVABLE)
+ capc->migratetype != MIGRATE_MOVABLE)
return false;
- if (migratetype != capc->cc->migratetype)
- trace_mm_page_alloc_extfrag(page, capc->cc->order, order,
- capc->cc->migratetype, migratetype);
+ if (migratetype != capc->migratetype)
+ trace_mm_page_alloc_extfrag(page, capc->order, order,
+ capc->migratetype, migratetype);
capc->page = page;
return true;
@@ -4146,18 +4146,67 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
struct page *page = NULL;
unsigned long pflags;
unsigned int noreclaim_flag;
+ struct capture_control capc = {
+ .zone = NULL,
+ .migratetype = ac->migratetype,
+ .order = order,
+ .page = NULL,
+ };
+ int compact_order = order;
- if (!order)
+ /*
+ * If fallbacks are not permitted (defrag_mode), we either
+ * need to reclaim space in a block of matching type, or clear
+ * out an entire block to allow __rmqueue_claim() to convert.
+ *
+ * Reclaim by itself is primarily freeing space in movable
+ * blocks, since that's where the LRU pages live. So this
+ * works for movable requests, but not for others.
+ *
+ * For those, promote the order to help make blocks, instead
+ * of spinning in reclaim alone unproductively.
+ */
+ if ((alloc_flags & ALLOC_NOFRAGMENT) && ac->migratetype != MIGRATE_MOVABLE)
+ compact_order = max(order, pageblock_order);
+
+ if (!compact_order)
return NULL;
psi_memstall_enter(&pflags);
delayacct_compact_start();
+ fs_reclaim_acquire(gfp_mask);
noreclaim_flag = memalloc_noreclaim_save();
- *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
- prio, &page);
+ /*
+ * Make sure the structs are really initialized before we expose the
+ * capture control, in case we are interrupted and the interrupt handler
+ * frees a page.
+ */
+ barrier();
+ WRITE_ONCE(current->capture_control, &capc);
+
+ *compact_result = try_to_compact_pages(gfp_mask, compact_order,
+ alloc_flags, ac, prio, &capc);
+
+ /*
+ * Make sure we hide capture control first before we read the captured
+ * page pointer, otherwise an interrupt could free and capture a page
+ * and we would leak it.
+ */
+ WRITE_ONCE(current->capture_control, NULL);
+ page = READ_ONCE(capc.page);
+
+ /*
+ * Technically, it is also possible that compaction is skipped but
+ * the page is still captured out of luck(IRQ came and freed the page).
+ * Returning COMPACT_SUCCESS in such cases helps in properly accounting
+ * the COMPACT[STALL|FAIL] when compaction is skipped.
+ */
+ if (page)
+ *compact_result = COMPACT_SUCCESS;
memalloc_noreclaim_restore(noreclaim_flag);
+ fs_reclaim_release(gfp_mask);
psi_memstall_leave(&pflags);
delayacct_compact_end();
@@ -4182,7 +4231,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
struct zone *zone = page_zone(page);
zone->compact_blockskip_flush = false;
- compaction_defer_reset(zone, order, true);
+ compaction_defer_reset(zone, compact_order, true);
count_vm_event(COMPACTSUCCESS);
return page;
}
@@ -4422,9 +4471,14 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
struct page *page = NULL;
unsigned long pflags;
bool drained = false;
+ int reclaim_order = order;
+
+ /* Match the slowpath compaction promotion in __alloc_pages_direct_compact */
+ if ((alloc_flags & ALLOC_NOFRAGMENT) && ac->migratetype != MIGRATE_MOVABLE)
+ reclaim_order = max(order, pageblock_order);
psi_memstall_enter(&pflags);
- *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
+ *did_some_progress = __perform_reclaim(gfp_mask, reclaim_order, ac);
if (unlikely(!(*did_some_progress)))
goto out;
diff --git a/mm/page_owner.c b/mm/page_owner.c
index 2dddcb6510aa..d8d264eb01b0 100644
--- a/mm/page_owner.c
+++ b/mm/page_owner.c
@@ -513,6 +513,7 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
{
#ifdef CONFIG_MEMCG
unsigned long memcg_data;
+ struct obj_cgroup *objcg;
struct mem_cgroup *memcg;
bool online;
char name[80];
@@ -522,11 +523,14 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
if (!memcg_data || PageTail(page))
goto out_unlock;
- if (memcg_data & MEMCG_DATA_OBJEXTS)
+ if (memcg_data & MEMCG_DATA_OBJEXTS) {
ret += scnprintf(kbuf + ret, count - ret,
"Slab cache page\n");
+ goto out_unlock;
+ }
- memcg = page_memcg_check(page);
+ objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
+ memcg = objcg ? obj_cgroup_memcg(objcg) : NULL;
if (!memcg)
goto out_unlock;
@@ -534,7 +538,7 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
cgroup_name(memcg->css.cgroup, name, sizeof(name));
ret += scnprintf(kbuf + ret, count - ret,
"Charged %sto %smemcg %s\n",
- PageMemcgKmem(page) ? "(via objcg) " : "",
+ (memcg_data & MEMCG_DATA_KMEM) ? "(via objcg) " : "",
online ? "" : "offline ",
name);
out_unlock:
diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c
index bac2eb5de63d..d7670ba4147b 100644
--- a/mm/page_vma_mapped.c
+++ b/mm/page_vma_mapped.c
@@ -107,7 +107,13 @@ again:
static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr)
{
unsigned long pfn;
- pte_t ptent = ptep_get(pvmw->pte);
+ pte_t ptent;
+
+ if (is_vm_hugetlb_page(pvmw->vma))
+ ptent = huge_ptep_get(pvmw->vma->vm_mm, pvmw->address,
+ pvmw->pte);
+ else
+ ptent = ptep_get(pvmw->pte);
if (pvmw->flags & PVMW_MIGRATION) {
const softleaf_t entry = softleaf_from_pte(ptent);
diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index 5d87c632a255..d3bfece31933 100644
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -126,6 +126,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
pmd = pmd_offset(pud, addr);
do {
again:
+ walk->action = ACTION_SUBTREE;
next = pmd_addr_end(addr, end);
if (pmd_none(*pmd)) {
if (has_install)
@@ -138,8 +139,6 @@ again:
continue;
}
- walk->action = ACTION_SUBTREE;
-
/*
* This implies that each ->pmd_entry() handler
* needs to know about pmd_trans_huge() pmds
@@ -196,6 +195,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
pud = pud_offset(p4d, addr);
do {
again:
+ walk->action = ACTION_SUBTREE;
next = pud_addr_end(addr, end);
if (pud_none(*pud)) {
if (has_install)
@@ -208,8 +208,6 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
continue;
}
- walk->action = ACTION_SUBTREE;
-
if (ops->pud_entry)
err = ops->pud_entry(pud, addr, next, walk);
if (err)
diff --git a/mm/rmap.c b/mm/rmap.c
index 1c77d5dc06e9..143cf80d95f4 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -2095,14 +2095,19 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
/* Unexpected PMD-mapped THP? */
VM_BUG_ON_FOLIO(!pvmw.pte, folio);
- /*
- * Handle PFN swap PTEs, such as device-exclusive ones, that
- * actually map pages.
- */
- pteval = ptep_get(pvmw.pte);
+ address = pvmw.address;
+ if (folio_test_hugetlb(folio)) {
+ pteval = huge_ptep_get(mm, address, pvmw.pte);
+ } else {
+ pteval = ptep_get(pvmw.pte);
+ }
if (likely(pte_present(pteval))) {
pfn = pte_pfn(pteval);
} else {
+ /*
+ * Handle PFN swap PTEs, such as device-exclusive ones,
+ * that actually map pages.
+ */
const softleaf_t entry = softleaf_from_pte(pteval);
pfn = softleaf_to_pfn(entry);
@@ -2110,7 +2115,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
}
subpage = folio_page(folio, pfn - folio_pfn(folio));
- address = pvmw.address;
anon_exclusive = folio_test_anon(folio) &&
PageAnonExclusive(subpage);
@@ -2501,14 +2505,18 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
/* Unexpected PMD-mapped THP? */
VM_BUG_ON_FOLIO(!pvmw.pte, folio);
- /*
- * Handle PFN swap PTEs, such as device-exclusive ones, that
- * actually map pages.
- */
- pteval = ptep_get(pvmw.pte);
+ address = pvmw.address;
+ if (folio_test_hugetlb(folio))
+ pteval = huge_ptep_get(mm, address, pvmw.pte);
+ else
+ pteval = ptep_get(pvmw.pte);
if (likely(pte_present(pteval))) {
pfn = pte_pfn(pteval);
} else {
+ /*
+ * Handle PFN swap PTEs, such as device-exclusive ones,
+ * that actually map pages.
+ */
const softleaf_t entry = softleaf_from_pte(pteval);
pfn = softleaf_to_pfn(entry);
@@ -2516,7 +2524,6 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
}
subpage = folio_page(folio, pfn - folio_pfn(folio));
- address = pvmw.address;
anon_exclusive = folio_test_anon(folio) &&
PageAnonExclusive(subpage);
diff --git a/mm/slub.c b/mm/slub.c
index 0337e60db5ac..5e88c520522b 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -333,14 +333,20 @@ enum track_item { TRACK_ALLOC, TRACK_FREE };
#ifdef SLAB_SUPPORTS_SYSFS
static int sysfs_slab_add(struct kmem_cache *);
+static int __init slab_kset_init(void);
+static void __init slab_sysfs_process_aliases(void);
#else
static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
+static inline int slab_kset_init(void) { return 0; }
+static inline void slab_sysfs_process_aliases(void) { }
#endif
#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
static void debugfs_slab_add(struct kmem_cache *);
+static void __init slab_debugfs_root_init(void);
#else
static inline void debugfs_slab_add(struct kmem_cache *s) { }
+static inline void slab_debugfs_root_init(void) { }
#endif
enum add_mode {
@@ -5147,7 +5153,7 @@ void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp,
* simply flush and free it.
*/
if (!barn || data_race(barn->nr_full) >= MAX_FULL_SHEAVES ||
- refill_sheaf(s, sheaf, gfp)) {
+ refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) {
sheaf_flush_unused(s, sheaf);
free_empty_sheaf(s, sheaf);
return;
@@ -9734,28 +9740,20 @@ int sysfs_slab_alias(struct kmem_cache *s, const char *name)
return 0;
}
-static int __init slab_sysfs_init(void)
+static int __init slab_kset_init(void)
{
- struct kmem_cache *s;
- int err;
-
- mutex_lock(&slab_mutex);
-
slab_kset = kset_create_and_add("slab", NULL, kernel_kobj);
if (!slab_kset) {
- mutex_unlock(&slab_mutex);
pr_err("Cannot register slab subsystem.\n");
return -ENOMEM;
}
- slab_state = FULL;
+ return 0;
+}
- list_for_each_entry(s, &slab_caches, list) {
- err = sysfs_slab_add(s);
- if (err)
- pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
- s->name);
- }
+static void __init slab_sysfs_process_aliases(void)
+{
+ int err;
while (alias_list) {
struct saved_alias *al = alias_list;
@@ -9767,12 +9765,41 @@ static int __init slab_sysfs_init(void)
al->name);
kfree(al);
}
+}
+#endif /* SLAB_SUPPORTS_SYSFS */
+
+#if defined(SLAB_SUPPORTS_SYSFS) || \
+ (defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS))
+static int __init slab_late_init(void)
+{
+ struct kmem_cache *s;
+ int err;
+ mutex_lock(&slab_mutex);
+
+ err = slab_kset_init();
+ if (err)
+ goto out;
+
+ slab_debugfs_root_init();
+ slab_state = FULL;
+
+ list_for_each_entry(s, &slab_caches, list) {
+ if (sysfs_slab_add(s))
+ pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
+ s->name);
+
+ if (s->flags & SLAB_STORE_USER)
+ debugfs_slab_add(s);
+ }
+
+ slab_sysfs_process_aliases();
+out:
mutex_unlock(&slab_mutex);
- return 0;
+ return err;
}
-late_initcall(slab_sysfs_init);
-#endif /* SLAB_SUPPORTS_SYSFS */
+late_initcall(slab_late_init);
+#endif
#if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
static int slab_debugfs_show(struct seq_file *seq, void *v)
@@ -9964,23 +9991,16 @@ static void debugfs_slab_add(struct kmem_cache *s)
void debugfs_slab_release(struct kmem_cache *s)
{
+ if (unlikely(!slab_debugfs_root))
+ return;
+
debugfs_lookup_and_remove(s->name, slab_debugfs_root);
}
-static int __init slab_debugfs_init(void)
+static void __init slab_debugfs_root_init(void)
{
- struct kmem_cache *s;
-
slab_debugfs_root = debugfs_create_dir("slab", NULL);
-
- list_for_each_entry(s, &slab_caches, list)
- if (s->flags & SLAB_STORE_USER)
- debugfs_slab_add(s);
-
- return 0;
-
}
-__initcall(slab_debugfs_init);
#endif
/*
* The /proc/slabinfo ABI
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index ebd3ac997f64..50df73b8f747 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -342,8 +342,8 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *
*
* Any initialization done here will be overwritten by memmap_init().
*
- * hugetlb_vmemmap_init() will take care of initialization after
- * memmap_init().
+ * hugetlb_bootmem_struct_page_init() will take care of initialization
+ * after memmap_init().
*/
p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 8538659acb9e..5bb24031e913 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -1893,11 +1893,11 @@ struct swap_info_struct *get_swap_device(swp_entry_t entry)
return si;
bad_nofile:
- pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val);
+ pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_file, entry.val);
out:
return NULL;
put_out:
- pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
+ pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
percpu_ref_put(&si->users);
return NULL;
}
@@ -2196,7 +2196,14 @@ void swap_free_hibernation_slot(swp_entry_t entry)
ci = swap_cluster_lock(si, offset);
__swap_cluster_put_entry(ci, offset % SWAPFILE_CLUSTER);
- __swap_cluster_free_entries(si, ci, offset % SWAPFILE_CLUSTER, 1);
+ /*
+ * A slot with a folio in the swap cache is freed when the folio
+ * leaves the cache, the same rule swap_put_entries_cluster() follows.
+ * Readahead can put a folio here, and freeing the slot now would
+ * leave that folio with no entry behind it.
+ */
+ if (!swp_tb_is_folio(__swap_table_get(ci, offset % SWAPFILE_CLUSTER)))
+ __swap_cluster_free_entries(si, ci, offset % SWAPFILE_CLUSTER, 1);
swap_cluster_unlock(ci);
/* In theory readahead might add it to the swap cache by accident */
@@ -3864,7 +3871,7 @@ int swap_dup_entry_direct(swp_entry_t entry)
si = swap_entry_to_info(entry);
if (WARN_ON_ONCE(!si)) {
- pr_err("%s%08lx\n", Bad_file, entry.val);
+ pr_err_ratelimited("%s%08lx\n", Bad_file, entry.val);
return -EINVAL;
}
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index f4fa227a8d7f..31c6425b5fdc 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -3360,7 +3360,7 @@ struct vm_struct *remove_vm_area(const void *addr)
static inline void set_area_direct_map(const struct vm_struct *area,
int (*set_direct_map)(struct page *page))
{
- int i;
+ unsigned long i;
/* HUGE_VMALLOC passes small pages to set_direct_map */
for (i = 0; i < area->nr_pages; i++)
@@ -3376,7 +3376,7 @@ static void vm_reset_perms(struct vm_struct *area)
unsigned long start = ULONG_MAX, end = 0;
unsigned int page_order = vm_area_page_order(area);
int flush_dmap = 0;
- int i;
+ unsigned long i;
/*
* Find the start and end range of the direct mappings to make sure that
@@ -3449,10 +3449,10 @@ void vfree_atomic(const void *addr)
* Caller is responsible for unmapping (vunmap_range) and KASAN
* poisoning before calling this.
*/
-static void vm_area_free_pages(struct vm_struct *vm, unsigned int start_idx,
- unsigned int end_idx)
+static void vm_area_free_pages(struct vm_struct *vm, unsigned long start_idx,
+ unsigned long end_idx)
{
- unsigned int i;
+ unsigned long i;
if (!(vm->flags & VM_MAP_PUT_PAGES)) {
for (i = start_idx; i < end_idx; i++)
@@ -3664,12 +3664,12 @@ static inline gfp_t vmalloc_gfp_adjust(gfp_t flags, const bool large)
return flags;
}
-static inline unsigned int
+static inline unsigned long
vm_area_alloc_pages(gfp_t gfp, int nid,
- unsigned int order, unsigned int nr_pages, struct page **pages)
+ unsigned int order, unsigned long nr_pages, struct page **pages)
{
- unsigned int nr_allocated = 0;
- unsigned int nr_remaining = nr_pages;
+ unsigned long nr_allocated = 0;
+ unsigned long nr_remaining = nr_pages;
unsigned int max_attempt_order = MAX_PAGE_ORDER;
struct page *page;
int i;
@@ -3717,7 +3717,7 @@ vm_area_alloc_pages(gfp_t gfp, int nid,
if (!order) {
while (nr_allocated < nr_pages) {
unsigned int nr, nr_pages_request;
- int i;
+ unsigned long i;
/*
* A maximum allowed request is hard-coded and is 100
@@ -3725,7 +3725,7 @@ vm_area_alloc_pages(gfp_t gfp, int nid,
* long preemption off scenario in the bulk-allocator
* so the range is [1:100].
*/
- nr_pages_request = min(100U, nr_pages - nr_allocated);
+ nr_pages_request = min(100UL, nr_pages - nr_allocated);
/* memory allocation should consider mempolicy, we can't
* wrongly use nearest node when nid == NUMA_NO_NODE,
@@ -3871,12 +3871,12 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
unsigned long addr = (unsigned long)area->addr;
unsigned long size = get_vm_area_size(area);
unsigned long array_size;
- unsigned int nr_small_pages = size >> PAGE_SHIFT;
+ unsigned long nr_small_pages = size >> PAGE_SHIFT;
unsigned int page_order;
unsigned int flags;
int ret;
- array_size = (unsigned long)nr_small_pages * sizeof(struct page *);
+ array_size = nr_small_pages * sizeof(struct page *);
/* __GFP_NOFAIL and "noblock" flags are mutually exclusive. */
if (!gfpflags_allow_blocking(gfp_mask))
@@ -4374,7 +4374,7 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align
}
if (size <= old_size) {
- unsigned int new_nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
+ unsigned long new_nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
/* Zero out "freed" memory, potentially for future realloc. */
if (want_init_on_free() || want_init_on_alloc(flags))
@@ -4403,7 +4403,7 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align
!(vm->flags & (VM_FLUSH_RESET_PERMS | VM_USERMAP)) &&
gfp_has_io_fs(flags)) {
unsigned long addr = (unsigned long)kasan_reset_tag(p);
- unsigned int old_nr_pages = vm->nr_pages;
+ unsigned long old_nr_pages = vm->nr_pages;
/*
* Use the node lock to synchronize with concurrent
@@ -4416,16 +4416,13 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align
spin_unlock(&vn->busy.lock);
/* Notify kmemleak of the reduced allocation size before unmapping. */
- kmemleak_free_part(
- (void *)addr + ((unsigned long)new_nr_pages
- << PAGE_SHIFT),
- (unsigned long)(old_nr_pages - new_nr_pages)
- << PAGE_SHIFT);
+ kmemleak_free_part((void *)addr +
+ (new_nr_pages << PAGE_SHIFT),
+ (old_nr_pages - new_nr_pages)
+ << PAGE_SHIFT);
- vunmap_range(addr + ((unsigned long)new_nr_pages
- << PAGE_SHIFT),
- addr + ((unsigned long)old_nr_pages
- << PAGE_SHIFT));
+ vunmap_range(addr + (new_nr_pages << PAGE_SHIFT),
+ addr + (old_nr_pages << PAGE_SHIFT));
vm_area_free_pages(vm, new_nr_pages, old_nr_pages);
}
@@ -5250,7 +5247,7 @@ bool vmalloc_dump_obj(void *object)
struct vmap_area *va;
struct vmap_node *vn;
unsigned long addr;
- unsigned int nr_pages;
+ unsigned long nr_pages;
addr = PAGE_ALIGN((unsigned long) object);
vn = addr_to_node(addr);
@@ -5270,7 +5267,7 @@ bool vmalloc_dump_obj(void *object)
nr_pages = vm->nr_pages;
spin_unlock(&vn->busy.lock);
- pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n",
+ pr_cont(" %lu-page vmalloc region starting at %#lx allocated at %pS\n",
nr_pages, addr, caller);
return true;
@@ -5288,16 +5285,17 @@ bool vmalloc_dump_obj(void *object)
static void show_numa_info(struct seq_file *m, struct vm_struct *v,
unsigned int *counters)
{
- unsigned int nr;
unsigned int step = 1U << vm_area_page_order(v);
+ unsigned long i;
+ unsigned int nr;
if (!counters)
return;
memset(counters, 0, nr_node_ids * sizeof(unsigned int));
- for (nr = 0; nr < v->nr_pages; nr += step)
- counters[page_to_nid(v->pages[nr])] += step;
+ for (i = 0; i < v->nr_pages; i += step)
+ counters[page_to_nid(v->pages[i])] += step;
for_each_node_state(nr, N_HIGH_MEMORY)
if (counters[nr])
seq_printf(m, " N%u=%u", nr, counters[nr]);
@@ -5355,7 +5353,7 @@ static int vmalloc_info_show(struct seq_file *m, void *p)
seq_printf(m, " %pS", v->caller);
if (v->nr_pages)
- seq_printf(m, " pages=%d", v->nr_pages);
+ seq_printf(m, " pages=%lu", v->nr_pages);
if (v->phys_addr)
seq_printf(m, " phys=%pa", &v->phys_addr);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 56708d1d2dfd..ee3efc6213f4 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -198,6 +198,13 @@ struct scan_control {
*/
int vm_swappiness = 60;
+static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg)
+{
+ if (sc->proactive && sc->proactive_swappiness)
+ return *sc->proactive_swappiness;
+ return mem_cgroup_swappiness(memcg);
+}
+
#ifdef CONFIG_MEMCG
/* Returns true for reclaim through cgroup limits or cgroup interfaces. */
@@ -238,13 +245,6 @@ static bool writeback_throttling_sane(struct scan_control *sc)
#endif
return false;
}
-
-static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg)
-{
- if (sc->proactive && sc->proactive_swappiness)
- return *sc->proactive_swappiness;
- return mem_cgroup_swappiness(memcg);
-}
#else
static bool cgroup_reclaim(struct scan_control *sc)
{
@@ -260,11 +260,6 @@ static bool writeback_throttling_sane(struct scan_control *sc)
{
return true;
}
-
-static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg)
-{
- return READ_ONCE(vm_swappiness);
-}
#endif
static void set_task_reclaim_state(struct task_struct *task,
@@ -4573,7 +4568,6 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent,
static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc,
int tier_idx)
{
- bool success;
int gen = folio_lru_gen(folio);
int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
@@ -4585,15 +4579,9 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
- /* unevictable */
- if (!folio_evictable(folio)) {
- success = lru_gen_del_folio(lruvec, folio, true);
- VM_WARN_ON_ONCE_FOLIO(!success, folio);
- folio_set_unevictable(folio);
- lruvec_add_folio(lruvec, folio);
- __count_vm_events(UNEVICTABLE_PGCULLED, delta);
- return true;
- }
+ /* unevictable: let it through and the generic path will cull it */
+ if (!folio_evictable(folio))
+ return false;
/* promoted */
if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
@@ -4843,11 +4831,9 @@ retry:
list_for_each_entry_safe_reverse(folio, next, &list, lru) {
DEFINE_MIN_SEQ(lruvec);
- if (!folio_evictable(folio)) {
- list_del(&folio->lru);
- folio_putback_lru(folio);
+ /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
+ if (!folio_evictable(folio))
continue;
- }
/* retry folios that may have missed folio_rotate_reclaimable() */
if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&
@@ -5927,7 +5913,7 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
}
}
- cond_resched();
+ cond_resched_tasks_rcu_qs();
if (nr_reclaimed < nr_to_reclaim || proportional_reclaim)
continue;
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 83f5820c45f9..41b03a9ecec2 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -478,6 +478,11 @@ static int get_size_class_index(int size)
return min_t(int, ZS_SIZE_CLASSES - 1, idx);
}
+static struct size_class *lookup_size_class(struct zs_pool *pool, size_t size)
+{
+ return pool->size_class[get_size_class_index(size + ZS_HANDLE_SIZE)];
+}
+
static inline void class_stat_add(struct size_class *class, int type,
unsigned long cnt)
{
@@ -1021,7 +1026,7 @@ unsigned int zs_lookup_class_index(struct zs_pool *pool, unsigned int size)
{
struct size_class *class;
- class = pool->size_class[get_size_class_index(size)];
+ class = lookup_size_class(pool, size);
return class->index;
}
@@ -1311,9 +1316,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp,
if (!handle)
return (unsigned long)ERR_PTR(-ENOMEM);
- /* extra space in chunk to keep the handle */
- size += ZS_HANDLE_SIZE;
- class = pool->size_class[get_size_class_index(size)];
+ class = lookup_size_class(pool, size);
/* class->lock effectively protects the zpage migration */
spin_lock(&class->lock);
diff --git a/mm/zswap.c b/mm/zswap.c
index 761cd699e0a3..0bdb999e46b6 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -1356,11 +1356,12 @@ static void shrink_worker(struct work_struct *w)
} while (memcg && !mem_cgroup_tryget_online(memcg));
spin_unlock(&zswap_shrink_lock);
- if (!memcg) {
- /*
- * Continue shrinking without incrementing failures if
- * we found candidate memcgs in the last tree walk.
- */
+ /*
+ * A NULL memcg ends a full hierarchy pass (except when memcg is
+ * disabled, where it is always NULL: fall through to the root LRU).
+ * Count a failure only if the last pass found no candidates.
+ */
+ if (!memcg && !mem_cgroup_disabled()) {
if (!attempts && ++failures == MAX_RECLAIM_RETRIES)
break;
@@ -1379,7 +1380,7 @@ static void shrink_worker(struct work_struct *w)
* and failures.
*/
if (ret == -ENOENT)
- continue;
+ goto resched;
++attempts;
if (ret && ++failures == MAX_RECLAIM_RETRIES)
generated by cgit v1.2.3 (git 2.39.1) at 2026-09-08 19:31:09 +0000