1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
|
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* sparse.h:
*
* mm/ internal sparse and sparse-vmemmap declarations
*/
#ifndef __MM_SPARSE_H
#define __MM_SPARSE_H
#include <linux/mmzone.h>
#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
static inline unsigned int section_order(const struct mem_section *section)
{
return section->order;
}
static inline void section_set_order(struct mem_section *section, unsigned int order)
{
VM_WARN_ON(section_order(section) && order && section_order(section) != order);
section->order = order;
}
static inline void section_set_order_range(unsigned long pfn, unsigned long nr_pages,
unsigned int order)
{
unsigned long section_nr = pfn_to_section_nr(pfn);
if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION))
return;
for (unsigned long i = 0; i < nr_pages / PAGES_PER_SECTION; i++)
section_set_order(__nr_to_section(section_nr + i), order);
}
static inline unsigned int pfn_to_section_order(unsigned long pfn)
{
return section_order(__pfn_to_section(pfn));
}
#else
static inline unsigned int section_order(const struct mem_section *section)
{
return 0;
}
static inline void section_set_order(struct mem_section *section, unsigned int order)
{
}
static inline void section_set_order_range(unsigned long pfn, unsigned long nr_pages,
unsigned int order)
{
}
static inline unsigned int pfn_to_section_order(unsigned long pfn)
{
return 0;
}
#endif
static inline bool vmemmap_optimizable_pfn(unsigned long pfn)
{
const unsigned int order = pfn_to_section_order(pfn);
const unsigned long nr_pages = 1UL << order;
if (!is_power_of_2(sizeof(struct page)))
return false;
return (pfn & (nr_pages - 1)) >= VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES;
}
static inline bool vmemmap_optimizable_order(unsigned int order)
{
if (!IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP))
return false;
if (!is_power_of_2(sizeof(struct page)))
return false;
return order >= VMEMMAP_OPTIMIZATION_MIN_ORDER;
}
/*
* mm/sparse.c
*/
#ifdef CONFIG_SPARSEMEM
void sparse_init(void);
void sparse_sections_init(void);
int sparse_index_init(unsigned long section_nr, int nid);
static inline void sparse_init_one_section(struct mem_section *ms,
unsigned long pnum, struct page *mem_map,
struct mem_section_usage *usage, unsigned long flags)
{
unsigned long coded_mem_map;
BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT);
/*
* We encode the start PFN of the section into the mem_map such that
* page_to_pfn() on !CONFIG_SPARSEMEM_VMEMMAP can simply subtract it
* from the page pointer to obtain the PFN.
*/
coded_mem_map = (unsigned long)(mem_map - section_nr_to_pfn(pnum));
VM_WARN_ON_ONCE(coded_mem_map & ~SECTION_MAP_MASK);
ms->section_mem_map &= ~SECTION_MAP_MASK;
ms->section_mem_map |= coded_mem_map;
ms->section_mem_map |= flags | SECTION_HAS_MEM_MAP;
ms->usage = usage;
}
static inline void __section_mark_present(struct mem_section *ms,
unsigned long section_nr)
{
if (section_nr > __highest_present_section_nr)
__highest_present_section_nr = section_nr;
ms->section_mem_map |= SECTION_MARKED_PRESENT;
}
static inline size_t mem_section_usage_size(void)
{
return struct_size_t(struct mem_section_usage, pageblock_flags,
BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS));
}
static inline bool section_vmemmap_optimizable(const struct mem_section *ms)
{
return vmemmap_optimizable_order(section_order(ms));
}
#else
static inline void sparse_init(void) {}
static inline void sparse_sections_init(void) {}
#endif /* CONFIG_SPARSEMEM */
/*
* mm/sparse-vmemmap.c
*/
#ifdef CONFIG_SPARSEMEM_VMEMMAP
void sparse_init_subsection_map(void);
int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
struct vmem_altmap *altmap, struct dev_pagemap *pgmap);
#else
static inline void sparse_init_subsection_map(void) {}
static inline int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
{
return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE);
}
#endif /* CONFIG_SPARSEMEM_VMEMMAP */
#endif /* __MM_SPARSE_H */
|
