1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
|
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_PREEMPT_H
#define __ASM_PREEMPT_H
#include <asm/rmwcc.h>
#include <asm/percpu.h>
#include <linux/static_call_types.h>
DECLARE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count);
/*
* We use the MSB for PREEMPT_NEED_RESCHED mostly because it is available.
*/
#define PREEMPT_NEED_RESCHED (~(((unsigned long)-1L) >> 1))
#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
#define __pc_dec "decq"
#define __pc_op(op, ...) raw_cpu_##op##_8(__VA_ARGS__)
#else
#define __pc_dec "decl"
#define __pc_op(op, ...) raw_cpu_##op##_4(__VA_ARGS__)
#endif
/*
* We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such
* that a decrement hitting 0 means we can and should reschedule.
*/
#define PREEMPT_ENABLED (0 + PREEMPT_NEED_RESCHED)
/*
* We mask the PREEMPT_NEED_RESCHED bit so as not to confuse all current users
* that think a non-zero value indicates we cannot preempt.
*/
static __always_inline int preempt_count(void)
{
return __pc_op(read, __preempt_count) & ~PREEMPT_NEED_RESCHED;
}
/*
* unsigned long preempt count parameter works for both 32bit and 64bit cases:
*
* - For 32bit, "int" (the return of preempt_count()) and "unsigned long" have
* the same size.
* - For 64bit, the effective bits of a preempt count sit in 32bit, and we
* preserve the NEED_RESCHED bit from the old count.
*/
static __always_inline void preempt_count_set(unsigned long pc)
{
unsigned long old, new;
old = __pc_op(read, __preempt_count);
do {
new = (old & PREEMPT_NEED_RESCHED) |
(pc & ~PREEMPT_NEED_RESCHED);
} while (!__pc_op(try_cmpxchg, __preempt_count, &old, new));
}
/*
* must be macros to avoid header recursion hell
*/
#define init_task_preempt_count(p) do { } while (0)
#define init_idle_preempt_count(p, cpu) do { \
per_cpu(__preempt_count, (cpu)) = PREEMPT_DISABLED; \
} while (0)
/*
* We fold the NEED_RESCHED bit into the preempt count such that
* preempt_enable() can decrement and test for needing to reschedule with a
* single instruction.
*
* We invert the actual bit, so that when the decrement hits 0 we know we both
* need to resched (the bit is cleared) and can resched (no preempt count).
*/
static __always_inline void set_preempt_need_resched(void)
{
__pc_op(and, __preempt_count, ~PREEMPT_NEED_RESCHED);
}
static __always_inline void clear_preempt_need_resched(void)
{
__pc_op(or, __preempt_count, PREEMPT_NEED_RESCHED);
}
static __always_inline bool test_preempt_need_resched(void)
{
return !(__pc_op(read, __preempt_count) & PREEMPT_NEED_RESCHED);
}
/*
* The various preempt_count add/sub methods
*/
static __always_inline void __preempt_count_add(int val)
{
__pc_op(add, __preempt_count, val);
}
static __always_inline void __preempt_count_sub(int val)
{
__pc_op(add, __preempt_count, -val);
}
static __always_inline int __preempt_count_add_return(int val)
{
return __pc_op(add_return, __preempt_count, val);
}
static __always_inline int __preempt_count_sub_return(int val)
{
return __pc_op(add_return, __preempt_count, -val);
}
/*
* Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to reschedule
* a decrement which hits zero means we have no preempt_count and should
* reschedule.
*/
static __always_inline bool __preempt_count_dec_and_test(void)
{
return GEN_UNARY_RMWcc(__pc_dec, __my_cpu_var(__preempt_count), e,
__percpu_arg([var]));
}
/*
* Returns true when we need to resched and can (barring IRQ state).
*/
static __always_inline bool should_resched(int preempt_offset)
{
return unlikely(__pc_op(read, __preempt_count) == preempt_offset);
}
#ifdef CONFIG_PREEMPTION
extern asmlinkage void preempt_schedule(void);
extern asmlinkage void preempt_schedule_thunk(void);
#define preempt_schedule_dynamic_enabled preempt_schedule_thunk
#define preempt_schedule_dynamic_disabled NULL
extern asmlinkage void preempt_schedule_notrace(void);
extern asmlinkage void preempt_schedule_notrace_thunk(void);
#define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace_thunk
#define preempt_schedule_notrace_dynamic_disabled NULL
#ifdef CONFIG_PREEMPT_DYNAMIC
DECLARE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled);
#define __preempt_schedule() \
do { \
__STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule); \
asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule) : ASM_CALL_CONSTRAINT); \
} while (0)
DECLARE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled);
#define __preempt_schedule_notrace() \
do { \
__STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule_notrace); \
asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule_notrace) : ASM_CALL_CONSTRAINT); \
} while (0)
#else /* PREEMPT_DYNAMIC */
#define __preempt_schedule() \
asm volatile ("call preempt_schedule_thunk" : ASM_CALL_CONSTRAINT);
#define __preempt_schedule_notrace() \
asm volatile ("call preempt_schedule_notrace_thunk" : ASM_CALL_CONSTRAINT);
#endif /* PREEMPT_DYNAMIC */
#endif /* PREEMPTION */
#undef __pc_op
#undef __pc_dec
#endif /* __ASM_PREEMPT_H */
|
