1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_STRING_64_H
#define _ASM_X86_STRING_64_H
#ifdef __KERNEL__
#include <linux/jump_label.h>
/* Written 2002 by Andi Kleen */
/* Even with __builtin_ the compiler may decide to use the out of line
function. */
#if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY)
#include <linux/kmsan_string.h>
#endif
#define __HAVE_ARCH_MEMCPY 1
extern void *memcpy(void *to, const void *from, size_t len);
extern void *__memcpy(void *to, const void *from, size_t len);
#define __HAVE_ARCH_MEMSET
void *memset(void *s, int c, size_t n);
void *__memset(void *s, int c, size_t n);
KCFI_REFERENCE(__memset);
/*
* KMSAN needs to instrument as much code as possible. Use C versions of
* memsetXX() from lib/string.c under KMSAN.
*/
#if !defined(CONFIG_KMSAN)
#define __HAVE_ARCH_MEMSET16
static inline void *memset16(uint16_t *s, uint16_t v, size_t n)
{
const auto s0 = s;
asm volatile (
"rep stosw"
: "+D" (s), "+c" (n)
: "a" (v)
: "memory"
);
return s0;
}
#define __HAVE_ARCH_MEMSET32
static inline void *memset32(uint32_t *s, uint32_t v, size_t n)
{
const auto s0 = s;
asm volatile (
"rep stosl"
: "+D" (s), "+c" (n)
: "a" (v)
: "memory"
);
return s0;
}
#define __HAVE_ARCH_MEMSET64
static inline void *memset64(uint64_t *s, uint64_t v, size_t n)
{
const auto s0 = s;
asm volatile (
"rep stosq"
: "+D" (s), "+c" (n)
: "a" (v)
: "memory"
);
return s0;
}
#endif
#define __HAVE_ARCH_MEMMOVE
void *memmove(void *dest, const void *src, size_t count);
void *__memmove(void *dest, const void *src, size_t count);
KCFI_REFERENCE(__memmove);
int memcmp(const void *cs, const void *ct, size_t count);
size_t strlen(const char *s);
char *strcpy(char *dest, const char *src);
char *strcat(char *dest, const char *src);
int strcmp(const char *cs, const char *ct);
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
#define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1
void __memcpy_flushcache(void *dst, const void *src, size_t cnt);
static __always_inline void movnti_4(void *dst, const void *src)
{
asm volatile("movntil %1, %0"
: "=m"(*(u32 *)dst)
: "r"(*(const u32 *)src)
: "memory");
}
static __always_inline void movnti_8(void *dst, const void *src)
{
asm volatile("movntiq %1, %0"
: "=m"(*(u64 *)dst)
: "r"(*(const u64 *)src)
: "memory");
}
static __always_inline void movnti_16(void *dst, const void *src)
{
movnti_8(dst, src);
movnti_8(dst + 8, src + 8);
}
static __always_inline void movnti_32(void *dst, const void *src)
{
movnti_16(dst, src);
movnti_16(dst + 16, src + 16);
}
static __always_inline void movnti_64(void *dst, const void *src)
{
movnti_32(dst, src);
movnti_32(dst + 32, src + 32);
}
static __always_inline void memcpy_flushcache(void *dst, const void *src,
size_t cnt)
{
if (!__builtin_constant_p(cnt))
return __memcpy_flushcache(dst, src, cnt);
/*
* The relevant fixed-size copies here are the x86_64 struct page sizes:
* 64, 80, and 96 bytes. Keep 32-byte and 48-byte copies inline as well
* instead of sending those nearby fixed-size cases back to
* __memcpy_flushcache().
*/
switch (cnt) {
case 4: movnti_4(dst, src); break;
case 8: movnti_8(dst, src); break;
case 16: movnti_16(dst, src); break;
case 32: movnti_32(dst, src); break;
case 48: movnti_32(dst, src); movnti_16(dst + 32, src + 32); break;
case 64: movnti_64(dst, src); break;
case 80: movnti_64(dst, src); movnti_16(dst + 64, src + 64); break;
case 96: movnti_64(dst, src); movnti_32(dst + 64, src + 64); break;
default: __memcpy_flushcache(dst, src, cnt); break;
}
}
#define memcpy_nontemporal memcpy_nontemporal
/*
* Reuse the existing x86 flushcache backend as the non-temporal copy
* primitive.
*/
static __always_inline void memcpy_nontemporal(void *dst, const void *src,
size_t cnt)
{
memcpy_flushcache(dst, src, cnt);
}
#endif
#endif /* __KERNEL__ */
#endif /* _ASM_X86_STRING_64_H */
|
