summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-09-06 13:49:44 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2026-09-06 13:49:44 -0700
commit2beb1b31a12b57e19cd5c82ea6d54e56520605e8 (patch)
tree622b0fd0b7fd41bdf9240873cb6d7514c248330b
parent88405f0ad1d5c680afe3ea0ce9345fa9e1deaac8 (diff)
parent536b523b407397c8d3967c020ce7aad70a0ea030 (diff)
downloadlinux-stable-2beb1b31a12b57e19cd5c82ea6d54e56520605e8.tar.gz
linux-stable-2beb1b31a12b57e19cd5c82ea6d54e56520605e8.zip
Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf
Pull bpf fixes from Alexei Starovoitov: "This mainly contains verifier fixes that address bugs reported by Nicholas Carlini. - Fix incorrect non-NULL inference in pointer comparisons: pointer types that may be NULL at runtime, pointers with unbounded offsets, JMP32 comparisons with zero, and imprecise zero registers (Eduard Zingerman) - Fix precision tracking for half-dead zero spills, ld_abs/ld_ind implicit subprog exit, bpf_loop() callbacks, linked scalar ids and NULL call arguments (Eduard Zingerman) - Reject BPF_PSEUDO_FUNC reference to the main program, fix zero extension of arena 32-bit cmpxchg, don't rewrite bpf_fastcall patterns entered by a jump (Eduard Zingerman) - Fix percpu map update and BPF_F_CPU validation with sparse CPU IDs (Hui Su) - Fix NULL-ptr-derefs in bpf_snprintf_btf() for void and VAR types, and reject key-less BTF for hash maps (Jiayuan Chen) - Various fixes (Kumar Kartikeya Dwivedi): - Fix out-of-bounds access in disassembler on invalid LDSX instruction - mark siginfo of signal tracepoints as scalar and sched_process_wait argument as nullable - mark faultable stack helpers as sleepable - reject tail calls and legacy packet loads from callbacks - enforce rbtree callback lock restrictions for resilient locks - require MEM_PERCPU for percpu kptr stores - clear NON_OWN_REF after RCU protection ends - mark NULL kptr stores precise - preserve inner map identity in callback frames - reject non-scalar bpf_loop() iteration counts - Fix trampoline allocation slowdown on x86 by using EXECMEM_MODULE_DATA (Mike Rapoport) - Keep bpf_refcount_acquire() nullable for borrowed RCU kptrs and reject untrusted allocated-object pointers (Ning Ding) - Fix special fields handling in recycled rhtab elements (Nuoqi Gui, Yuan Chen)" * tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: (86 commits) bpf, riscv: Make arena support depend on ZACAS selftests/bpf: Test pointer bpf_loop iteration count rejection bpf: Reject non-scalar bpf_loop iteration counts bpf: use mark_arg_precision() in check_mem_size_reg() bpf: propagate mark_chain_precision() errors out of loop_flag_is_zero() selftests/bpf: precision of a NULL global subprogram BTF_ID argument bpf: mark a NULL BTF_ID argument of a global subprogram precise selftests/bpf: precision of a NULL kfunc argument bpf: mark a NULL kfunc argument precise selftests/bpf: precision of a NULL global subprogram memory argument bpf: mark a NULL memory argument of a call precise selftests/bpf: precision of a NULL helper argument bpf: mark a NULL call argument precise selftests/bpf: Test inner map identities in callbacks bpf: Preserve inner map identity in callback frames selftests/bpf: Test imprecise scalar kptr stores bpf: Mark NULL kptr stores precise selftests/bpf: Test rhtab kptr cancellation semantics bpf: Cancel special fields when recycling rhtab elements selftests/bpf: Test timer field on recycled rhtab element ...
-rw-r--r--arch/riscv/net/bpf_jit_comp64.c10
-rw-r--r--arch/x86/net/bpf_jit_comp.c8
-rw-r--r--include/linux/bpf.h3
-rw-r--r--include/linux/bpf_verifier.h16
-rw-r--r--include/linux/filter.h1
-rw-r--r--kernel/bpf/arraymap.c5
-rw-r--r--kernel/bpf/backtrack.c62
-rw-r--r--kernel/bpf/bpf_iter.c2
-rw-r--r--kernel/bpf/btf.c27
-rw-r--r--kernel/bpf/cfg.c3
-rw-r--r--kernel/bpf/core.c5
-rw-r--r--kernel/bpf/disasm.c50
-rw-r--r--kernel/bpf/fixups.c7
-rw-r--r--kernel/bpf/hashtab.c29
-rw-r--r--kernel/bpf/local_storage.c5
-rw-r--r--kernel/bpf/percpu_freelist.c35
-rw-r--r--kernel/bpf/percpu_freelist.h1
-rw-r--r--kernel/bpf/stackmap.c2
-rw-r--r--kernel/bpf/states.c11
-rw-r--r--kernel/bpf/syscall.c2
-rw-r--r--kernel/bpf/verifier.c259
-rw-r--r--tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c59
-rw-r--r--tools/testing/selftests/bpf/prog_tests/rhash.c6
-rw-r--r--tools/testing/selftests/bpf/prog_tests/rhash_timer.c141
-rw-r--r--tools/testing/selftests/bpf/prog_tests/snprintf_btf.c79
-rw-r--r--tools/testing/selftests/bpf/prog_tests/timer_mim.c29
-rw-r--r--tools/testing/selftests/bpf/progs/iters.c39
-rw-r--r--tools/testing/selftests/bpf/progs/map_kptr_fail.c37
-rw-r--r--tools/testing/selftests/bpf/progs/percpu_alloc_fail.c59
-rw-r--r--tools/testing/selftests/bpf/progs/preempt_lock.c52
-rw-r--r--tools/testing/selftests/bpf/progs/raw_tp_null_fail.c53
-rw-r--r--tools/testing/selftests/bpf/progs/rbtree_fail.c84
-rw-r--r--tools/testing/selftests/bpf/progs/rcu_read_lock.c6
-rw-r--r--tools/testing/selftests/bpf/progs/refcounted_kptr.c61
-rw-r--r--tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c150
-rw-r--r--tools/testing/selftests/bpf/progs/rhash.c112
-rw-r--r--tools/testing/selftests/bpf/progs/rhash_timer.c98
-rw-r--r--tools/testing/selftests/bpf/progs/snprintf_btf_void.c24
-rw-r--r--tools/testing/selftests/bpf/progs/tailcall_callback.c16
-rw-r--r--tools/testing/selftests/bpf/progs/task_local_data.bpf.h3
-rw-r--r--tools/testing/selftests/bpf/progs/test_bpf_nf.c8
-rw-r--r--tools/testing/selftests/bpf/progs/timer_mim_reject.c84
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_async_cb_context.c98
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_bounds.c41
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c110
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_cfg.c14
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c29
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c24
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c17
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c193
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_ld_ind.c96
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c78
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_precision.c100
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_scalar_ids.c13
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_spill_fill.c40
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_subprog_precision.c63
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_zext.c26
-rw-r--r--tools/testing/selftests/bpf/verifier/pseudo_func.c45
58 files changed, 2555 insertions, 175 deletions
diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c
index 74efe4b138d2..151031e97a24 100644
--- a/arch/riscv/net/bpf_jit_comp64.c
+++ b/arch/riscv/net/bpf_jit_comp64.c
@@ -2128,7 +2128,15 @@ bool bpf_jit_supports_ptr_xchg(void)
bool bpf_jit_supports_arena(void)
{
- return true;
+ /*
+ * The arena range tree uses kmalloc_nolock(), which needs
+ * cmpxchg128, provided by ZACAS on riscv.
+ */
+#ifdef system_has_cmpxchg128
+ return system_has_cmpxchg128();
+#else
+ return false;
+#endif
}
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 1a9fb530adc3..2853e87797a7 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -13,6 +13,7 @@
#include <linux/bpf_verifier.h>
#include <linux/memory.h>
#include <linux/sort.h>
+#include <linux/execmem.h>
#include <asm/extable.h>
#include <asm/ftrace.h>
#include <asm/set_memory.h>
@@ -3818,15 +3819,16 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
*
* We cannot use kvmalloc here, because we need image to be in
* module memory range.
- * Since it must be writable use bpf_jit_alloc_exec_rw().
+ * Since it must be writable use execmem_alloc(EXECMEM_MODULE_DATA)
+ * that returns writable memory in the module address space.
*/
- image = bpf_jit_alloc_exec_rw(PAGE_SIZE);
+ image = execmem_alloc(EXECMEM_MODULE_DATA, PAGE_SIZE);
if (!image)
return -ENOMEM;
ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, image,
m, flags, tnodes, func_addr);
- bpf_jit_free_exec(image);
+ execmem_free(image);
return ret;
}
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index ffa5626411ac..e57af902560c 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -894,6 +894,7 @@ enum bpf_arg_type {
ARG_PTR_TO_CTX, /* pointer to context */
ARG_ANYTHING, /* any (initialized) argument is ok */
+ ARG_SCALAR, /* scalar argument */
ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */
ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */
ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */
@@ -4209,7 +4210,7 @@ static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 all
return -EINVAL;
cpu = flags >> 32;
- if ((flags & BPF_F_CPU) && cpu >= num_possible_cpus())
+ if ((flags & BPF_F_CPU) && (cpu >= nr_cpu_ids || !cpu_possible(cpu)))
return -ERANGE;
}
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 5fad59fdab0d..36b65797877d 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -706,6 +706,8 @@ struct bpf_insn_aux_data {
*/
u32 calls_callback:1;
u32 indirect_target:1; /* if it is an indirect jump target */
+ /* true if some jump or call instruction targets this instruction */
+ u32 jump_target:1;
/*
* CFG strongly connected component this instruction belongs to,
* zero if it is a singleton SCC.
@@ -1142,6 +1144,16 @@ static inline void mark_jmp_point(struct bpf_verifier_env *env, int idx)
env->insn_aux_data[idx].jmp_point = true;
}
+static inline void mark_jump_target(struct bpf_verifier_env *env, int idx)
+{
+ env->insn_aux_data[idx].jump_target = true;
+}
+
+static inline bool bpf_is_jump_target(struct bpf_verifier_env *env, int insn_idx)
+{
+ return env->insn_aux_data[insn_idx].jump_target;
+}
+
static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)
{
struct bpf_verifier_state *cur = env->cur_state;
@@ -1369,7 +1381,9 @@ static inline bool bpf_type_has_unsafe_modifiers(u32 type)
static inline bool type_is_ptr_alloc_obj(u32 type)
{
- return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC;
+ return base_type(type) == PTR_TO_BTF_ID &&
+ type_flag(type) & MEM_ALLOC &&
+ !(type_flag(type) & PTR_UNTRUSTED);
}
static inline bool type_is_non_owning_ref(u32 type)
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 4a9bc6a848f2..39decde7fc73 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1376,7 +1376,6 @@ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
void bpf_jit_binary_free(struct bpf_binary_header *hdr);
u64 bpf_jit_alloc_exec_limit(void);
void *bpf_jit_alloc_exec(unsigned long size);
-void *bpf_jit_alloc_exec_rw(unsigned long size);
void bpf_jit_free_exec(void *addr);
void bpf_jit_free(struct bpf_prog *fp);
struct bpf_binary_header *
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index ef315b168b29..0ce26b538075 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -436,7 +436,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
void __percpu *pptr;
void *ptr, *val;
u32 size;
- int cpu;
+ int cpu, off = 0;
if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS))
/* unknown flags */
@@ -468,9 +468,10 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
}
for_each_possible_cpu(cpu) {
ptr = per_cpu_ptr(pptr, cpu);
- val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu;
+ val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off;
copy_map_value(map, ptr, val);
bpf_obj_cancel_fields(map, ptr);
+ off += size;
}
unlock:
rcu_read_unlock();
diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c
index a2b18a9f1694..47282ffeeaf9 100644
--- a/kernel/bpf/backtrack.c
+++ b/kernel/bpf/backtrack.c
@@ -520,37 +520,34 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
return -EFAULT;
}
} else if (opcode == BPF_EXIT) {
- bool r0_precise;
+ bool from_subprog_call, r0_precise;
+
+ /* BPF_EXIT in subprog or callback always returns
+ * right after the call instruction, so by checking
+ * whether the instruction at subseq_idx-1 is subprog
+ * call or not we can distinguish actual exit from
+ * *subprog* from exit from *callback*. In the former
+ * case, we need to propagate r0 precision, if
+ * necessary. In the former we never do that.
+ */
+ from_subprog_call = subseq_idx - 1 >= 0 &&
+ bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]);
+
+ r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0);
/* Backtracking to a nested function call, 'idx' is a part of
* the inner frame 'subseq_idx' is a part of the outer frame.
* In case of a regular function call, instructions giving
* precision to registers R1-R5 should have been found already.
- * In case of a callback, it is ok to have R1-R5 marked for
- * backtracking, as these registers are set by the function
- * invoking callback.
+ * In case of a callback from bpf_loop(), R{1,4} in the calling
+ * frame would be set as precise and that is correct.
*/
- if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx))
- for (i = BPF_REG_1; i <= BPF_REG_5; i++)
- bt_clear_reg(bt, i);
- if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
+ if (from_subprog_call && (bt_reg_mask(bt) & BPF_REGMASK_ARGS)) {
verifier_bug(env, "backtracking exit unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
- /* BPF_EXIT in subprog or callback always returns
- * right after the call instruction, so by checking
- * whether the instruction at subseq_idx-1 is subprog
- * call or not we can distinguish actual exit from
- * *subprog* from exit from *callback*. In the former
- * case, we need to propagate r0 precision, if
- * necessary. In the former we never do that.
- */
- r0_precise = subseq_idx - 1 >= 0 &&
- bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) &&
- bt_is_reg_set(bt, BPF_REG_0);
-
bt_clear_reg(bt, BPF_REG_0);
if (bt_subprog_enter(bt))
return -EFAULT;
@@ -582,16 +579,29 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
*/
}
} else if (class == BPF_LD) {
- if (!bt_is_reg_set(bt, dreg))
- return 0;
- bt_clear_reg(bt, dreg);
/* It's ld_imm64 or ld_abs or ld_ind.
* For ld_imm64 no further tracking of precision
* into parent is necessary
*/
- if (mode == BPF_IND || mode == BPF_ABS)
- /* to be analyzed */
- return -ENOTSUPP;
+ if (mode == BPF_IMM) {
+ bt_clear_reg(bt, dreg);
+ return 0;
+ }
+ /*
+ * BPF_{IND,ABS} are modelled as two branches:
+ * - fallthrough;
+ * - implicit subprogram exit.
+ * It is necessary to switch current frame if
+ * implicit subprogram exit branch is backtracked.
+ */
+ if (mode == BPF_IND || mode == BPF_ABS) {
+ if (bt_is_reg_set(bt, dreg))
+ return -ENOTSUPP;
+ if (subseq_idx != idx + 1)
+ if (bt_subprog_enter(bt))
+ return -EFAULT;
+ return 0;
+ }
}
/* Propagate precision marks to linked registers, to account for
* registers marked as precise in this function.
diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c
index 14a5fdfa0421..b40eb404adab 100644
--- a/kernel/bpf/bpf_iter.c
+++ b/kernel/bpf/bpf_iter.c
@@ -754,7 +754,7 @@ const struct bpf_func_proto bpf_loop_proto = {
.func = bpf_loop,
.gpl_only = false,
.ret_type = RET_INTEGER,
- .arg1_type = ARG_ANYTHING,
+ .arg1_type = ARG_SCALAR,
.arg2_type = ARG_PTR_TO_FUNC,
.arg3_type = ARG_PTR_TO_STACK_OR_NULL,
.arg4_type = ARG_ANYTHING,
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index da36d4b9d31a..9f33e95d5741 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -2911,14 +2911,29 @@ static void btf_modifier_show(const struct btf *btf,
else
t = btf_type_skip_modifiers(btf, type_id, NULL);
- btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
+ /*
+ * A modifier can resolve to void, which has no show op; print a
+ * placeholder rather than dereferencing NULL.
+ */
+ if (!btf_type_ops(t))
+ btf_df_show(btf, t, type_id, data, bits_offset, show);
+ else
+ btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
}
static void btf_var_show(const struct btf *btf, const struct btf_type *t,
u32 type_id, void *data, u8 bits_offset,
struct btf_show *show)
{
- t = btf_type_id_resolve(btf, &type_id);
+ /*
+ * btf_type_id_resolve() dereferences btf->resolved_ids, which is NULL
+ * for a base BTF (e.g. the vmlinux BTF that bpf_snprintf_btf() uses).
+ * Resolve the var's type directly in that case.
+ */
+ if (btf->resolved_ids)
+ t = btf_type_id_resolve(btf, &type_id);
+ else
+ t = btf_type_skip_modifiers(btf, t->type, &type_id);
btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
}
@@ -6657,6 +6672,10 @@ struct bpf_raw_tp_null_args {
static const struct bpf_raw_tp_null_args raw_tp_null_args[] = {
/* sched */
{ "sched_pi_setprio", 0x10 },
+ /*
+ * do_wait() passes NULL for wait4(-1) and waitid(P_ALL).
+ */
+ { "sched_process_wait", 0x1 },
/* ... from sched_numa_pair_template event class */
{ "sched_stick_numa", 0x100 },
{ "sched_swap_numa", 0x100 },
@@ -6717,6 +6736,9 @@ static const struct bpf_raw_tp_null_args raw_tp_null_args[] = {
{ "rxrpc_resend", 0x10 },
{ "rxrpc_tq", 0x10 },
{ "rxrpc_client", 0x1 },
+ /* signal */
+ { "signal_generate", 0x20 },
+ { "signal_deliver", 0x20 },
/* skb */
{"kfree_skb", 0x1000},
/* sunrpc */
@@ -8727,6 +8749,7 @@ BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int
const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = {
.func = bpf_btf_find_by_name_kind,
.gpl_only = false,
+ .might_sleep = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
.arg2_type = ARG_MEM_SIZE,
diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c
index 0f13c13f4133..842c7d1eabcc 100644
--- a/kernel/bpf/cfg.c
+++ b/kernel/bpf/cfg.c
@@ -125,6 +125,7 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)
/* mark branch target for state pruning */
mark_prune_point(env, w);
mark_jmp_point(env, w);
+ mark_jump_target(env, w);
}
if (insn_state[w] == 0) {
@@ -403,6 +404,7 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env)
}
mark_jmp_point(env, w);
+ mark_jump_target(env, w);
/* EXPLORED || DISCOVERED */
if (insn_state[w])
@@ -564,6 +566,7 @@ static int visit_insn(int t, struct bpf_verifier_env *env)
mark_prune_point(env, t + off + 1);
mark_jmp_point(env, t + off + 1);
+ mark_jump_target(env, t + off + 1);
return ret;
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index d55e737ed75a..8b294dfc1ad4 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1128,11 +1128,6 @@ void *bpf_jit_alloc_exec(unsigned long size)
return execmem_alloc(EXECMEM_BPF, size);
}
-void *bpf_jit_alloc_exec_rw(unsigned long size)
-{
- return execmem_alloc_rw(EXECMEM_BPF, size);
-}
-
void bpf_jit_free_exec(void *addr)
{
execmem_free(addr);
diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c
index 50b3ca5149a0..3ce8d74b0e40 100644
--- a/kernel/bpf/disasm.c
+++ b/kernel/bpf/disasm.c
@@ -7,6 +7,9 @@
#include "disasm.h"
+/* Only defined by the non-UAPI linux/filter.h, which this file cannot use. */
+#define BPF_PROBE_ATOMIC 0xe0
+
#define __BPF_FUNC_STR_FN(x) [BPF_FUNC_ ## x] = __stringify(bpf_ ## x)
static const char * const func_id_str[] = {
__BPF_FUNC_MAPPER(__BPF_FUNC_STR_FN)
@@ -226,57 +229,57 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
insn->imm);
}
} else if (class == BPF_STX) {
+ const char *probe_pfx = BPF_MODE(insn->code) == BPF_PROBE_ATOMIC ? "probe " : "";
+ bool atomic = BPF_MODE(insn->code) == BPF_ATOMIC ||
+ BPF_MODE(insn->code) == BPF_PROBE_ATOMIC;
+
if (BPF_MODE(insn->code) == BPF_MEM)
verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = r%d",
insn->code,
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg,
insn->off, insn->src_reg);
- else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
+ else if (atomic &&
(insn->imm == BPF_ADD || insn->imm == BPF_AND ||
insn->imm == BPF_OR || insn->imm == BPF_XOR)) {
- verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d",
- insn->code,
+ verbose(cbs->private_data, "(%02x) %slock *(%s *)(r%d %+d) %s r%d",
+ insn->code, probe_pfx,
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg, insn->off,
bpf_alu_string[BPF_OP(insn->imm) >> 4],
insn->src_reg);
- } else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
+ } else if (atomic &&
(insn->imm == (BPF_ADD | BPF_FETCH) ||
insn->imm == (BPF_AND | BPF_FETCH) ||
insn->imm == (BPF_OR | BPF_FETCH) ||
insn->imm == (BPF_XOR | BPF_FETCH))) {
- verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)",
- insn->code, insn->src_reg,
+ verbose(cbs->private_data, "(%02x) %sr%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)",
+ insn->code, probe_pfx, insn->src_reg,
BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
bpf_atomic_alu_string[BPF_OP(insn->imm) >> 4],
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg, insn->off, insn->src_reg);
- } else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
- insn->imm == BPF_CMPXCHG) {
- verbose(cbs->private_data, "(%02x) r0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)",
- insn->code,
+ } else if (atomic && insn->imm == BPF_CMPXCHG) {
+ verbose(cbs->private_data, "(%02x) %sr0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)",
+ insn->code, probe_pfx,
BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg, insn->off,
insn->src_reg);
- } else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
- insn->imm == BPF_XCHG) {
- verbose(cbs->private_data, "(%02x) r%d = atomic%s_xchg((%s *)(r%d %+d), r%d)",
- insn->code, insn->src_reg,
+ } else if (atomic && insn->imm == BPF_XCHG) {
+ verbose(cbs->private_data, "(%02x) %sr%d = atomic%s_xchg((%s *)(r%d %+d), r%d)",
+ insn->code, probe_pfx, insn->src_reg,
BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg, insn->off, insn->src_reg);
- } else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
- insn->imm == BPF_LOAD_ACQ) {
- verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))",
- insn->code, insn->dst_reg,
+ } else if (atomic && insn->imm == BPF_LOAD_ACQ) {
+ verbose(cbs->private_data, "(%02x) %sr%d = load_acquire((%s *)(r%d %+d))",
+ insn->code, probe_pfx, insn->dst_reg,
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->src_reg, insn->off);
- } else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
- insn->imm == BPF_STORE_REL) {
- verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)",
- insn->code,
+ } else if (atomic && insn->imm == BPF_STORE_REL) {
+ verbose(cbs->private_data, "(%02x) %sstore_release((%s *)(r%d %+d), r%d)",
+ insn->code, probe_pfx,
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg, insn->off, insn->src_reg);
} else {
@@ -295,7 +298,8 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
verbose(cbs->private_data, "BUG_st_%02x", insn->code);
}
} else if (class == BPF_LDX) {
- if (BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) {
+ if ((BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) ||
+ (BPF_MODE(insn->code) == BPF_MEMSX && BPF_SIZE(insn->code) == BPF_DW)) {
verbose(cbs->private_data, "BUG_ldx_%02x", insn->code);
return;
}
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 65b441e4a351..52d3cec33672 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -13,10 +13,15 @@
#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
+/*
+ * Matches BPF_PROBE_ATOMIC too: bpf_convert_ctx_accesses() rewrites arena
+ * atomics before bpf_opt_subreg_zext_lo32_rnd_hi32() runs.
+ */
static bool is_cmpxchg_insn(const struct bpf_insn *insn)
{
return BPF_CLASS(insn->code) == BPF_STX &&
- BPF_MODE(insn->code) == BPF_ATOMIC &&
+ (BPF_MODE(insn->code) == BPF_ATOMIC ||
+ BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) &&
insn->imm == BPF_CMPXCHG;
}
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index c2796e8d29ea..6f331c80130d 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -530,6 +530,9 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf,
{
struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
+ if (btf_type_is_void(key_type))
+ return -EINVAL;
+
if (htab_is_prealloc(htab))
return 0;
/*
@@ -1025,7 +1028,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
} else {
u32 size = round_up(htab->map.value_size, 8);
void *val;
- int cpu;
+ int cpu, off = 0;
if (map_flags & BPF_F_CPU) {
cpu = map_flags >> 32;
@@ -1037,9 +1040,10 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
for_each_possible_cpu(cpu) {
ptr = per_cpu_ptr(pptr, cpu);
- val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu;
+ val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off;
copy_map_value(&htab->map, ptr, val);
bpf_obj_cancel_fields(&htab->map, ptr);
+ off += size;
}
}
}
@@ -2864,16 +2868,6 @@ static int rhtab_map_alloc_check(union bpf_attr *attr)
return htab_map_alloc_check(attr);
}
-static void rhtab_check_and_free_fields(struct bpf_rhtab *rhtab,
- struct rhtab_elem *elem)
-{
- if (IS_ERR_OR_NULL(rhtab->map.record))
- return;
-
- bpf_obj_free_fields(rhtab->map.record,
- rhtab_elem_value(elem, rhtab->map.key_size));
-}
-
static void rhtab_mem_dtor(void *obj, void *ctx)
{
struct htab_btf_record *hrec = ctx;
@@ -2963,8 +2957,8 @@ static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, v
rhtab_read_elem_value(&rhtab->map, copy, elem, flags);
check_and_init_map_value(&rhtab->map, copy);
}
- /* Release internal structs: kptr, bpf_timer, task_work, wq */
- rhtab_check_and_free_fields(rhtab, elem);
+ bpf_obj_cancel_fields(&rhtab->map,
+ rhtab_elem_value(elem, rhtab->map.key_size));
bpf_mem_cache_free_rcu(&rhtab->ma, elem);
return 0;
}
@@ -3005,7 +2999,6 @@ static int rhtab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, void
static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *elem, void *value,
u64 map_flags)
{
- struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
void *old_val = rhtab_elem_value(elem, map->key_size);
if (map_flags & BPF_NOEXIST)
@@ -3025,7 +3018,7 @@ static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *el
* kptrs/etc. still sit in the slot. Cancel them after the copy
* to match arraymap's update semantics.
*/
- rhtab_check_and_free_fields(rhtab, elem);
+ bpf_obj_cancel_fields(map, old_val);
return 0;
}
@@ -3066,7 +3059,6 @@ static long rhtab_map_update_elem(struct bpf_map *map, void *key, void *value, u
memcpy(elem->data, key, map->key_size);
copy_map_value(map, rhtab_elem_value(elem, map->key_size), value);
- check_and_init_map_value(map, rhtab_elem_value(elem, map->key_size));
/* Prevent deadlock for NMI programs attempting to take bucket lock */
bpf_disable_instrumentation();
@@ -3110,6 +3102,9 @@ static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf,
{
struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
+ if (btf_type_is_void(key_type))
+ return -EINVAL;
+
return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor);
}
diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c
index 23267213a17f..83cd527a2542 100644
--- a/kernel/bpf/local_storage.c
+++ b/kernel/bpf/local_storage.c
@@ -220,7 +220,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key,
struct bpf_cgroup_storage *storage;
void *val;
u32 size;
- int cpu;
+ int cpu, off = 0;
if ((u32)map_flags & ~(BPF_ANY | BPF_EXIST | BPF_F_CPU | BPF_F_ALL_CPUS))
return -EINVAL;
@@ -245,8 +245,9 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key,
}
size = round_up(_map->value_size, 8);
for_each_possible_cpu(cpu) {
- val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu;
+ val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off;
copy_map_value(_map, per_cpu_ptr(storage->percpu_buf, cpu), val);
+ off += size;
}
unlock:
rcu_read_unlock();
diff --git a/kernel/bpf/percpu_freelist.c b/kernel/bpf/percpu_freelist.c
index 632762b57299..06ce588d13a3 100644
--- a/kernel/bpf/percpu_freelist.c
+++ b/kernel/bpf/percpu_freelist.c
@@ -17,6 +17,8 @@ int pcpu_freelist_init(struct pcpu_freelist *s)
raw_res_spin_lock_init(&head->lock);
head->first = NULL;
}
+ raw_res_spin_lock_init(&s->extralist.lock);
+ s->extralist.first = NULL;
return 0;
}
@@ -46,22 +48,28 @@ void __pcpu_freelist_push(struct pcpu_freelist *s,
struct pcpu_freelist_node *node)
{
struct pcpu_freelist_head *head;
- int cpu;
+ int cpu, this_cpu;
if (___pcpu_freelist_push(this_cpu_ptr(s->freelist), node))
return;
+ this_cpu = raw_smp_processor_id();
while (true) {
- for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) {
- if (cpu == raw_smp_processor_id())
+ for_each_cpu_wrap(cpu, cpu_possible_mask, this_cpu) {
+ if (cpu == this_cpu)
continue;
+
head = per_cpu_ptr(s->freelist, cpu);
- if (raw_res_spin_lock(&head->lock))
- continue;
- pcpu_freelist_push_node(head, node);
- raw_res_spin_unlock(&head->lock);
- return;
+ if (___pcpu_freelist_push(head, node))
+ return;
}
+
+ /*
+ * Push cannot fail. Use the extra list when none of the
+ * per-CPU freelists can accept the node.
+ */
+ if (___pcpu_freelist_push(&s->extralist, node))
+ return;
}
}
@@ -117,6 +125,17 @@ static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s)
}
raw_res_spin_unlock(&head->lock);
}
+
+ /* Per-CPU lists are empty or unavailable, try the extra list. */
+ head = &s->extralist;
+ if (!READ_ONCE(head->first))
+ return NULL;
+ if (raw_res_spin_lock(&head->lock))
+ return NULL;
+ node = head->first;
+ if (node)
+ WRITE_ONCE(head->first, node->next);
+ raw_res_spin_unlock(&head->lock);
return node;
}
diff --git a/kernel/bpf/percpu_freelist.h b/kernel/bpf/percpu_freelist.h
index 914798b74967..980cf2884fd2 100644
--- a/kernel/bpf/percpu_freelist.h
+++ b/kernel/bpf/percpu_freelist.h
@@ -14,6 +14,7 @@ struct pcpu_freelist_head {
struct pcpu_freelist {
struct pcpu_freelist_head __percpu *freelist;
+ struct pcpu_freelist_head extralist;
};
struct pcpu_freelist_node {
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index a839041e0d00..d09d4c3fe547 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -875,6 +875,7 @@ BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, si
const struct bpf_func_proto bpf_get_stack_sleepable_proto = {
.func = bpf_get_stack_sleepable,
.gpl_only = true,
+ .might_sleep = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
@@ -928,6 +929,7 @@ BPF_CALL_4(bpf_get_task_stack_sleepable, struct task_struct *, task, void *, buf
const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = {
.func = bpf_get_task_stack_sleepable,
.gpl_only = false,
+ .might_sleep = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_BTF_ID,
.arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index 4e6aafad33bd..66fb11b6c6a7 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -445,22 +445,19 @@ static void __clean_func_state(struct bpf_verifier_env *env,
struct bpf_reg_state *spill = &st->stack[i].spilled_ptr;
if (lo_live && stype == STACK_SPILL) {
- u8 val = STACK_MISC;
-
if (spill->type != SCALAR_VALUE)
continue;
-
/*
- * 8 byte spill of scalar 0 where half slot is dead
- * should become STACK_ZERO in lo 4 bytes.
+ * Can't replace with STACK_ZERO, because
+ * that requires bpf_mark_chain_precision().
*/
if (bpf_register_is_null(spill))
- val = STACK_ZERO;
+ continue;
for (j = 0; j < 4; j++) {
u8 *t = &st->stack[i].slot_type[j];
if (*t == STACK_SPILL)
- *t = val;
+ *t = STACK_MISC;
}
}
bpf_mark_reg_not_init(env, spill);
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 6874ba1424af..c7bc9ba9b331 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -6568,6 +6568,7 @@ EXPORT_SYMBOL_NS(kern_sys_bpf, "BPF_INTERNAL");
static const struct bpf_func_proto bpf_sys_bpf_proto = {
.func = bpf_sys_bpf,
.gpl_only = false,
+ .might_sleep = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_ANYTHING,
.arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
@@ -6593,6 +6594,7 @@ BPF_CALL_1(bpf_sys_close, u32, fd)
static const struct bpf_func_proto bpf_sys_close_proto = {
.func = bpf_sys_close,
.gpl_only = false,
+ .might_sleep = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_ANYTHING,
};
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index a68e5435ca5f..72a3f5998dd2 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -352,9 +352,18 @@ static bool reg_not_null(struct bpf_verifier_env *env, const struct bpf_reg_stat
if (type_may_be_null(type))
return false;
+ /*
+ * The types below guarantee a non-NULL base, an unbounded offset can
+ * still wrap base + offset to zero.
+ */
+ if (reg_smin(reg) <= -BPF_MAX_VAR_OFF || reg_smax(reg) >= BPF_MAX_VAR_OFF)
+ return false;
+
type = base_type(type);
return type == PTR_TO_SOCKET ||
type == PTR_TO_TCP_SOCK ||
+ type == PTR_TO_XDP_SOCK ||
+ type == PTR_TO_BUF ||
type == PTR_TO_MAP_VALUE ||
type == PTR_TO_MAP_KEY ||
type == PTR_TO_SOCK_COMMON ||
@@ -4237,6 +4246,15 @@ static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx)
return mark_chain_precision_batch(env, env->cur_state);
}
+static int mark_arg_precision(struct bpf_verifier_env *env, argno_t argno)
+{
+ int regno = reg_from_argno(argno);
+
+ if (regno >= 0)
+ return mark_chain_precision(env, regno);
+ return mark_stack_arg_precision(env, arg_idx_from_argno(argno));
+}
+
static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller,
int nargs, const char *callee_name, const struct btf *btf,
const struct btf_param *args)
@@ -4486,6 +4504,13 @@ static int map_kptr_match_type(struct bpf_verifier_env *env,
if (type_flag(reg->type) & ~perm_flags)
goto bad_type;
+ /*
+ * A BPF_KPTR_PERCPU field is read back as MEM_PERCPU, so the value
+ * stored in it must carry the same flag.
+ */
+ if ((kptr_field->type == BPF_KPTR_PERCPU) != !!(reg->type & MEM_PERCPU))
+ goto bad_type;
+
/* We need to verify reg->type and reg->btf, before accessing reg->btf */
reg_name = btf_type_name(reg->btf, reg->btf_id);
@@ -4692,8 +4717,15 @@ static int check_map_kptr_access(struct bpf_verifier_env *env,
return ret;
} else if (class == BPF_STX) {
val_reg = reg_state(env, value_regno);
- if (!bpf_register_is_null(val_reg) &&
- map_kptr_match_type(env, kptr_field, val_reg, value_regno))
+ if (bpf_register_is_null(val_reg)) {
+ /*
+ * This store is valid only because the scalar is known to be
+ * zero. Mark it precise so another scalar cannot be pruned
+ * against this state.
+ */
+ return mark_chain_precision(env, value_regno);
+ }
+ if (map_kptr_match_type(env, kptr_field, val_reg, value_regno))
return -EACCES;
} else if (class == BPF_ST) {
if (insn->imm) {
@@ -5300,6 +5332,15 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
if (!priv_stack_supported)
subprog[idx].priv_stack_mode = NO_PRIV_STACK;
process_func:
+ if (subprog[idx].has_ld_abs) {
+ for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) {
+ if (subprog[tmp].is_cb) {
+ verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n");
+ return -EINVAL;
+ }
+ }
+ }
+
/* protect against potential stack overflow that might happen when
* bpf2bpf calls get combined with tailcalls. Limit the caller's stack
* depth for such case down to 256 so that the worst case scenario
@@ -6020,7 +6061,13 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
return -EACCES;
}
- if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) &&
+ /*
+ * A fault-prone allocated object may still be read through a
+ * BPF_PROBE_MEM load after its lifetime protection ends. Writes
+ * through such pointers were rejected above.
+ */
+ if (type_is_alloc(reg->type) && !bpf_may_fault_on_deref(reg->type) &&
+ !type_is_non_owning_ref(reg->type) &&
!(reg->type & MEM_RCU) && !reg_is_referenced(env, reg)) {
verifier_bug(env, "allocated object must have a referenced id");
return -EFAULT;
@@ -7113,14 +7160,8 @@ static int check_mem_size_reg(struct bpf_verifier_env *env,
if (err && failure)
*failure = BPF_MEM_SIZE_FAIL_MEMORY;
- if (!err) {
- int regno = reg_from_argno(size_argno);
-
- if (regno >= 0)
- err = mark_chain_precision(env, regno);
- else
- err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno));
- }
+ if (!err)
+ err = mark_arg_precision(env, size_argno);
return err;
@@ -7137,7 +7178,7 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg
int size, err = 0;
if (bpf_register_is_null(reg))
- return 0;
+ return mark_arg_precision(env, argno);
if (known_memory)
*known_memory = true;
@@ -7398,10 +7439,14 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state
lock);
return -EINVAL;
}
+ /*
+ * Invalidate non-owning refs before RCU demotion clears their
+ * NON_OWN_REF flag.
+ */
+ invalidate_non_owning_refs(env);
+
if (!in_rcu_cs(env))
invalidate_rcu_protected_refs(env);
-
- invalidate_non_owning_refs(env);
}
return 0;
}
@@ -8166,6 +8211,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
[ARG_MEM_SIZE] = &scalar_types,
[ARG_MEM_SIZE_OR_ZERO] = &scalar_types,
[ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types,
+ [ARG_SCALAR] = &scalar_types,
[ARG_CONST_MAP_PTR] = &const_map_ptr_types,
[ARG_PTR_TO_CTX] = &context_types,
[ARG_PTR_TO_SOCK_COMMON] = &sock_types,
@@ -8717,11 +8763,15 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
return err;
}
- if (bpf_register_is_null(reg) && type_may_be_null(arg_type))
+ if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) {
/* A NULL register has a SCALAR_VALUE type, so skip
* type checking.
*/
+ err = mark_chain_precision(env, regno);
+ if (err)
+ return err;
goto skip_type_check;
+ }
/* arg_btf_id and arg_size are in a union. */
if (base_type(arg_type) == ARG_PTR_TO_BTF_ID ||
@@ -9501,7 +9551,7 @@ static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env)
bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({
if (reg->type & MEM_RCU) {
bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE);
- reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
+ reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL | NON_OWN_REF);
reg->type |= PTR_UNTRUSTED;
bpf_diag_mod_end(env);
}
@@ -9719,8 +9769,12 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
struct bpf_call_arg_meta meta;
int err;
- if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type))
+ if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type)) {
+ err = mark_arg_precision(env, argno);
+ if (err)
+ return err;
continue;
+ }
memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */
err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta,
@@ -9976,10 +10030,12 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env,
callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr;
+ callee->regs[BPF_REG_2].map_uid = caller->regs[BPF_REG_1].map_uid;
callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr;
+ callee->regs[BPF_REG_3].map_uid = caller->regs[BPF_REG_1].map_uid;
/* pointer to stack or null */
callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3];
@@ -10057,6 +10113,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env,
int insn_idx)
{
struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr;
+ u32 map_uid = caller->regs[BPF_REG_1].map_uid;
/* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn);
* callback_fn(struct bpf_map *map, void *key, void *value);
@@ -10064,14 +10121,17 @@ static int set_timer_callback_state(struct bpf_verifier_env *env,
callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
callee->regs[BPF_REG_1].map_ptr = map_ptr;
+ callee->regs[BPF_REG_1].map_uid = map_uid;
callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
callee->regs[BPF_REG_2].map_ptr = map_ptr;
+ callee->regs[BPF_REG_2].map_uid = map_uid;
callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
callee->regs[BPF_REG_3].map_ptr = map_ptr;
+ callee->regs[BPF_REG_3].map_uid = map_uid;
/* unused */
bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
@@ -10171,6 +10231,7 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env,
int insn_idx)
{
struct bpf_map *map_ptr = caller->regs[BPF_REG_3].map_ptr;
+ u32 map_uid = caller->regs[BPF_REG_3].map_uid;
/*
* callback_fn(struct bpf_map *map, void *key, void *value);
@@ -10178,14 +10239,17 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env,
callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
callee->regs[BPF_REG_1].map_ptr = map_ptr;
+ callee->regs[BPF_REG_1].map_uid = map_uid;
callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
callee->regs[BPF_REG_2].map_ptr = map_ptr;
+ callee->regs[BPF_REG_2].map_uid = map_uid;
callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
callee->regs[BPF_REG_3].map_ptr = map_ptr;
+ callee->regs[BPF_REG_3].map_uid = map_uid;
/* unused */
bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
@@ -10233,9 +10297,10 @@ static void account_current_path(struct bpf_verifier_env *env)
frame ? state->frame[frame - 1] : NULL);
}
-/* Are we currently verifying the callback for a rbtree helper that must
- * be called with lock held? If so, no need to complain about unreleased
- * lock
+/*
+ * Are we currently verifying the callback for an rbtree kfunc that must
+ * be called with a lock held, or one of that callback's subprogs? If so,
+ * no need to complain about an unreleased lock.
*/
static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)
{
@@ -10243,17 +10308,19 @@ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)
struct bpf_insn *insn = env->prog->insnsi;
struct bpf_func_state *callee;
int kfunc_btf_id;
+ u32 frame;
- if (!state->curframe)
- return false;
-
- callee = state->frame[state->curframe];
+ for (frame = state->curframe; frame; frame--) {
+ callee = state->frame[frame];
+ if (!callee->in_callback_fn)
+ continue;
- if (!callee->in_callback_fn)
- return false;
+ kfunc_btf_id = insn[callee->callsite].imm;
+ if (is_rbtree_lock_required_kfunc(kfunc_btf_id))
+ return true;
+ }
- kfunc_btf_id = insn[callee->callsite].imm;
- return is_rbtree_lock_required_kfunc(kfunc_btf_id);
+ return false;
}
static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg)
@@ -10630,33 +10697,45 @@ static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env)
return &env->insn_aux_data[env->insn_idx];
}
-static bool loop_flag_is_zero(struct bpf_verifier_env *env)
+/* Returns 1 if R4 is a known zero, 0 if it is not, a negative errno on error. */
+static int loop_flag_is_zero(struct bpf_verifier_env *env)
{
struct bpf_reg_state *reg = reg_state(env, BPF_REG_4);
- bool reg_is_null = bpf_register_is_null(reg);
+ int err;
- if (reg_is_null)
- mark_chain_precision(env, BPF_REG_4);
+ if (!bpf_register_is_null(reg))
+ return 0;
- return reg_is_null;
+ err = mark_chain_precision(env, BPF_REG_4);
+ if (err)
+ return err;
+ return 1;
}
-static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)
+static int update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)
{
struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state;
+ int flag_is_zero;
if (!state->initialized) {
+ flag_is_zero = loop_flag_is_zero(env);
+ if (flag_is_zero < 0)
+ return flag_is_zero;
state->initialized = 1;
- state->fit_for_inline = loop_flag_is_zero(env);
+ state->fit_for_inline = flag_is_zero;
state->callback_subprogno = subprogno;
- return;
+ return 0;
}
if (!state->fit_for_inline)
- return;
+ return 0;
- state->fit_for_inline = (loop_flag_is_zero(env) &&
+ flag_is_zero = loop_flag_is_zero(env);
+ if (flag_is_zero < 0)
+ return flag_is_zero;
+ state->fit_for_inline = (flag_is_zero &&
state->callback_subprogno == subprogno);
+ return 0;
}
/* Returns whether or not the given map can potentially elide
@@ -10868,6 +10947,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
verbose(env, "get_local_storage() doesn't support non-zero flags\n");
return -EINVAL;
}
+ err = mark_chain_precision(env, BPF_REG_2);
+ if (err)
+ return err;
break;
case BPF_FUNC_for_each_map_elem:
err = push_callback_call(env, insn, insn_idx, meta.subprogno,
@@ -10885,7 +10967,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
err = check_bpf_snprintf_call(env, regs);
break;
case BPF_FUNC_loop:
- update_loop_inline_state(env, meta.subprogno);
+ err = update_loop_inline_state(env, meta.subprogno);
+ if (err)
+ return err;
/* Verifier relies on R1 value to determine if bpf_loop() iteration
* is finished, thus mark it precise.
*/
@@ -11226,6 +11310,17 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
if (env->cur_state->curframe) {
struct bpf_verifier_state *branch;
+ /*
+ * A taken tail call is modeled as a return from the current
+ * frame. A callback frame cannot be left that way because
+ * prepare_func_exit() would apply its return contract to the
+ * unknown R0 synthesized below. Stack-depth validation rejects
+ * this construct anyway.
+ */
+ if (cur_func(env)->in_callback_fn) {
+ verbose(env, "cannot tail call within callback\n");
+ return -EINVAL;
+ }
mark_reg_scratched(env, BPF_REG_0);
branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);
if (IS_ERR(branch))
@@ -12651,8 +12746,12 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me
if (reg_is_referenced(env, reg))
update_ref_obj(&meta->ref_obj, reg);
- if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type))
+ if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) {
+ ret = mark_arg_precision(env, argno);
+ if (ret)
+ return ret;
continue;
+ }
if (is_kfunc_arg_map(btf, &args[i])) {
ref_id = *reg2btf_ids[CONST_PTR_TO_MAP];
@@ -13146,7 +13245,7 @@ check_ok:
bpf_diag_reg_type_plain(env, reg->type));
return -EINVAL;
}
- if (!type_is_non_owning_ref(reg->type))
+ if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg))
meta->arg_owning_ref = true;
rec = reg_btf_record(reg);
@@ -13228,6 +13327,11 @@ check_ok:
{
int flags = PROCESS_RES_LOCK;
+ if (in_rbtree_lock_required_cb(env)) {
+ verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n");
+ return -EACCES;
+ }
+
if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
verbose(env, "%s doesn't point to map value or allocated object\n",
reg_arg_name(env, argno));
@@ -14560,9 +14664,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
return -EINVAL;
}
- /* pointer types do not carry 32-bit bounds at the moment. */
- __mark_reg32_unbounded(dst_reg);
-
if (sanitize_needed(opcode)) {
ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
&info, false);
@@ -14570,6 +14671,14 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
return sanitize_err(env, insn, ret);
}
+ /*
+ * Pointer types do not carry 32-bit bounds at the moment. Blank r32
+ * only after sanitize_ptr_alu() may have snapshotted dst_reg into a
+ * speculative path: otherwise reg_bounds_sanity_check() might hit some
+ * constraints violations.
+ */
+ __mark_reg32_unbounded(dst_reg);
+
switch (opcode) {
case BPF_ADD:
/*
@@ -16295,6 +16404,13 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r
if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) {
u64 val;
+ /*
+ * The low 32 bits of a valid pointer may well be zero, hence
+ * nothing below applies to a 32-bit comparison.
+ */
+ if (is_jmp32)
+ return -1;
+
/* arrange that reg2 is a scalar, and reg1 is a pointer */
if (!is_reg_const(reg2, is_jmp32)) {
opcode = flip_opcode(opcode);
@@ -16856,6 +16972,16 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
return err;
}
+ /*
+ * Collect the linked registers before env->{true,false}_reg{1,2} setup,
+ * otherwise ids dropped by collect_linked_regs() would be resurrected
+ * when env->{true,false}_reg{1,2} are copied back.
+ */
+ if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id)
+ collect_linked_regs(env, this_branch, src_reg->id, &linked_regs);
+ if (dst_reg->type == SCALAR_VALUE && dst_reg->id)
+ collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs);
+
is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
env->false_reg1 = *dst_reg;
env->false_reg2 = *src_reg;
@@ -16910,10 +17036,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
* 'this_branch' and 'other_branch' share this history
* if parent state is created.
*/
- if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id)
- collect_linked_regs(env, this_branch, src_reg->id, &linked_regs);
- if (dst_reg->type == SCALAR_VALUE && dst_reg->id)
- collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs);
if (linked_regs.cnt > 1) {
err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs));
if (err)
@@ -16963,7 +17085,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
*/
if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X &&
__is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) &&
- type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) &&
base_type(src_reg->type) != PTR_TO_BTF_ID &&
base_type(dst_reg->type) != PTR_TO_BTF_ID) {
eq_branch_regs = NULL;
@@ -16979,9 +17100,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
break;
}
if (eq_branch_regs) {
- if (type_may_be_null(src_reg->type))
+ /* src == dst && dst != NULL => src != NULL */
+ if (reg_not_null(env, dst_reg) && type_may_be_null(src_reg->type))
mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]);
- else
+ /* src == dst && src != NULL => dst != NULL */
+ if (reg_not_null(env, src_reg) && type_may_be_null(dst_reg->type))
mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]);
}
}
@@ -16996,6 +17119,15 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
type_may_be_null(dst_reg->type) &&
((BPF_SRC(insn->code) == BPF_K && insn->imm == 0) ||
(BPF_SRC(insn->code) == BPF_X && bpf_register_is_null(src_reg)))) {
+ /*
+ * For BPF_X the zero is a property of this execution path,
+ * hence src_reg has to be precise.
+ */
+ if (BPF_SRC(insn->code) == BPF_X) {
+ err = mark_chain_precision(env, insn->src_reg);
+ if (err)
+ return err;
+ }
/* Mark all identical registers in each branch as either
* safe or unknown depending R == 0 or R != 0 conditional.
*/
@@ -17081,6 +17213,15 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
verbose(env, "callback function not static\n");
return -EINVAL;
}
+ /*
+ * When env->subprog_cnt == 1 this instruction won't be rewritten
+ * to hold a real function address. Assume that no usable program
+ * combines e.g. main and timer callback and just reject here.
+ */
+ if (subprogno == 0) {
+ verbose(env, "callback function cannot be the main program\n");
+ return -EINVAL;
+ }
dst_reg->type = PTR_TO_FUNC;
dst_reg->subprogno = subprogno;
@@ -17146,6 +17287,7 @@ static bool may_access_skb(enum bpf_prog_type type)
*/
static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
{
+ struct bpf_verifier_state *state = env->cur_state;
struct bpf_reg_state *regs = cur_regs(env);
static const int ctx_reg = BPF_REG_6;
u8 mode = BPF_MODE(insn->code);
@@ -17156,6 +17298,13 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
return -EINVAL;
}
+ for (i = state->curframe; i; i--) {
+ if (state->frame[i]->in_callback_fn) {
+ verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n");
+ return -EINVAL;
+ }
+ }
+
if (!env->ops->gen_ld_abs) {
verifier_bug(env, "gen_ld_abs is null");
return -EFAULT;
@@ -17623,6 +17772,10 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,
* r0 = *(u64 *)(r10 - 8); r0 += r1;
* r0 += r1; exit;
* exit;
+ *
+ * Both uses of the marks assume that a pattern is entered at its first
+ * spill and thus executes as a unit, hence a pattern is not grown past
+ * an instruction targeted by a jump.
*/
static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
struct bpf_subprog_info *subprog,
@@ -17661,6 +17814,10 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
for (i = 1, off = lowest_off; i <= ARRAY_SIZE(caller_saved); ++i, off += BPF_REG_SIZE) {
if (insn_idx - i < 0 || insn_idx + i >= env->prog->len)
break;
+ /* stx/ldx/call must not be a jump targets, a jump to the first stx is fine */
+ if (bpf_is_jump_target(env, insn_idx - i + 1) ||
+ bpf_is_jump_target(env, insn_idx + i))
+ break;
stx = &insns[insn_idx - i];
ldx = &insns[insn_idx + i];
/* must be a stack spill/fill pair */
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c b/tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c
new file mode 100644
index 000000000000..3248bccc3557
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <test_progs.h>
+#include <bpf/btf.h>
+
+/*
+ * A hash map with a key-less BTF (btf_key_type_id == 0) used to be accepted
+ * and then NULL-deref in btf_type_show() when dumped through bpffs. A fixed
+ * kernel rejects it at creation; verify that rejection, with a keyed positive
+ * control so the -EINVAL is about the missing key type and not some unrelated
+ * failure.
+ */
+static void check_keyless(int map_type, __u32 map_flags, int btf_fd, int val_id)
+{
+ LIBBPF_OPTS(bpf_map_create_opts, opts);
+ int map_fd;
+
+ opts.map_flags = map_flags;
+ opts.btf_fd = btf_fd;
+ opts.btf_value_type_id = val_id;
+
+ /* Positive control: the same map with a real key type is accepted. */
+ opts.btf_key_type_id = val_id;
+ map_fd = bpf_map_create(map_type, "keyed_map", 4, 4, 8, &opts);
+ if (!ASSERT_GE(map_fd, 0, "keyed create is accepted"))
+ return;
+ close(map_fd);
+
+ /* A key-less BTF must be rejected. */
+ opts.btf_key_type_id = 0;
+ map_fd = bpf_map_create(map_type, "keyless_map", 4, 4, 8, &opts);
+ ASSERT_EQ(map_fd, -EINVAL, "key-less create is rejected");
+ if (map_fd >= 0)
+ close(map_fd);
+}
+
+void test_btf_map_keyless(void)
+{
+ int btf_fd, val_id;
+ struct btf *btf;
+
+ btf = btf__new_empty();
+ if (!ASSERT_OK_PTR(btf, "btf__new_empty"))
+ return;
+
+ val_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED);
+ if (!ASSERT_GT(val_id, 0, "btf__add_int"))
+ goto out;
+
+ if (!ASSERT_OK(btf__load_into_kernel(btf), "btf__load_into_kernel"))
+ goto out;
+ btf_fd = btf__fd(btf);
+
+ if (test__start_subtest("hash"))
+ check_keyless(BPF_MAP_TYPE_HASH, 0, btf_fd, val_id);
+ if (test__start_subtest("rhash"))
+ check_keyless(BPF_MAP_TYPE_RHASH, BPF_F_NO_PREALLOC, btf_fd, val_id);
+out:
+ btf__free(btf);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/rhash.c b/tools/testing/selftests/bpf/prog_tests/rhash.c
index 98bb66907b7f..0641bd5b0a9e 100644
--- a/tools/testing/selftests/bpf/prog_tests/rhash.c
+++ b/tools/testing/selftests/bpf/prog_tests/rhash.c
@@ -172,6 +172,12 @@ void test_rhash(void)
if (test__start_subtest("test_rhash_delete_nonexistent"))
rhash_run("test_rhash_delete_nonexistent");
+ if (test__start_subtest("test_rhash_kptr_update"))
+ rhash_run("test_rhash_kptr_update");
+
+ if (test__start_subtest("test_rhash_kptr_delete"))
+ rhash_run("test_rhash_kptr_delete");
+
if (test__start_subtest("test_rhash_map_extra_presize"))
rhash_map_extra_presize();
diff --git a/tools/testing/selftests/bpf/prog_tests/rhash_timer.c b/tools/testing/selftests/bpf/prog_tests/rhash_timer.c
new file mode 100644
index 000000000000..3aad9fc02e06
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/rhash_timer.c
@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#define _GNU_SOURCE
+#include <sched.h>
+
+#include <test_progs.h>
+#include "rhash_timer.skel.h"
+
+#define MAX_ATTEMPTS 256
+#define RCU_SYNC_INTERVAL 64
+
+static int pin_to_first_cpu(cpu_set_t *old_mask)
+{
+ cpu_set_t new_mask;
+ int cpu;
+
+ if (sched_getaffinity(0, sizeof(*old_mask), old_mask))
+ return -errno;
+
+ for (cpu = 0; cpu < CPU_SETSIZE; cpu++)
+ if (CPU_ISSET(cpu, old_mask))
+ break;
+ if (cpu == CPU_SETSIZE)
+ return -EINVAL;
+
+ CPU_ZERO(&new_mask);
+ CPU_SET(cpu, &new_mask);
+ if (sched_setaffinity(0, sizeof(new_mask), &new_mask))
+ return -errno;
+ return 0;
+}
+
+static int update_timer_map(int map_fd, __u64 key)
+{
+ __u64 value[3] = {};
+
+ return bpf_map_update_elem(map_fd, &key, value, BPF_NOEXIST);
+}
+
+static int run_prog(int prog_fd, struct bpf_test_run_opts *opts)
+{
+ int err;
+
+ err = bpf_prog_test_run_opts(prog_fd, opts);
+ if (err)
+ return err;
+ return opts->retval;
+}
+
+void test_rhash_timer(void)
+{
+ LIBBPF_OPTS(bpf_test_run_opts, opts);
+ struct rhash_timer *skel = NULL;
+ cpu_set_t old_mask;
+ int map_fd = -1, arm_fd, cancel_fd;
+ bool affinity_set = false;
+ __u64 key = 1;
+ int attempt, err;
+
+ err = pin_to_first_cpu(&old_mask);
+ if (!ASSERT_OK(err, "pin_to_first_cpu"))
+ return;
+ affinity_set = true;
+
+ skel = rhash_timer__open_and_load();
+ if (!ASSERT_OK_PTR(skel, "open_and_load"))
+ goto out;
+
+ map_fd = bpf_map__fd(skel->maps.timer_map);
+ if (!ASSERT_GE(map_fd, 0, "timer_map fd"))
+ goto out;
+ arm_fd = bpf_program__fd(skel->progs.arm_deleted_timer);
+ if (!ASSERT_GE(arm_fd, 0, "arm_deleted_timer fd"))
+ goto out;
+ cancel_fd = bpf_program__fd(skel->progs.cancel_recycled_timer);
+ if (!ASSERT_GE(cancel_fd, 0, "cancel_recycled_timer fd"))
+ goto out;
+
+ err = update_timer_map(map_fd, key);
+ if (!ASSERT_OK(err, "seed_timer_map"))
+ goto out;
+
+ for (attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+ err = run_prog(arm_fd, &opts);
+ if (err) {
+ ASSERT_OK(err, "arm_deleted_timer");
+ goto out;
+ }
+ if (skel->bss->armed != attempt + 1) {
+ ASSERT_EQ(skel->bss->armed, attempt + 1, "armed");
+ goto out;
+ }
+ if (skel->bss->timer_init_err) {
+ ASSERT_OK(skel->bss->timer_init_err, "timer_init_err");
+ goto out;
+ }
+ if (skel->bss->timer_set_callback_err) {
+ ASSERT_OK(skel->bss->timer_set_callback_err,
+ "timer_set_callback_err");
+ goto out;
+ }
+ if (skel->bss->timer_start_err) {
+ ASSERT_OK(skel->bss->timer_start_err, "timer_start_err");
+ goto out;
+ }
+
+ if ((attempt + 1) % RCU_SYNC_INTERVAL == 0) {
+ err = kern_sync_rcu();
+ if (err) {
+ ASSERT_OK(err, "kern_sync_rcu");
+ goto out;
+ }
+ }
+
+ err = update_timer_map(map_fd, ++key);
+ if (err) {
+ ASSERT_OK(err, "replace_timer_map");
+ goto out;
+ }
+
+ err = run_prog(cancel_fd, &opts);
+ if (err) {
+ ASSERT_OK(err, "cancel_recycled_timer");
+ goto out;
+ }
+ if (skel->bss->timer_cancel_err) {
+ ASSERT_OK(skel->bss->timer_cancel_err, "timer_cancel_err");
+ goto out;
+ }
+ if (skel->bss->cancelled)
+ break;
+ }
+
+ ASSERT_GT(skel->bss->cancelled, 0, "preserved timer");
+out:
+ if (map_fd >= 0)
+ bpf_map_delete_elem(map_fd, &key);
+ rhash_timer__destroy(skel);
+ if (affinity_set)
+ sched_setaffinity(0, sizeof(old_mask), &old_mask);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c b/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c
index dd41b826be30..edce9c1b54fb 100644
--- a/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c
+++ b/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c
@@ -1,7 +1,9 @@
// SPDX-License-Identifier: GPL-2.0
#include <test_progs.h>
#include <linux/btf.h>
+#include <bpf/btf.h>
#include "netif_receive_skb.skel.h"
+#include "snprintf_btf_void.skel.h"
/* Demonstrate that bpf_snprintf_btf succeeds and that various data types
* are formatted correctly.
@@ -58,3 +60,80 @@ void serial_test_snprintf_btf(void)
cleanup:
netif_receive_skb__destroy(skel);
}
+
+/*
+ * bpf_snprintf_btf() renders a type_id taken straight from the vmlinux BTF.
+ * Two such type_ids used to NULL-deref in the BTF show path:
+ * - a "const void" (a modifier resolving to void) in btf_modifier_show()
+ * - a BTF_KIND_VAR in btf_var_show() (base BTF has no resolved_ids)
+ * A fixed kernel renders both without crashing.
+ */
+static long run(struct snprintf_btf_void *skel, __u32 type_id)
+{
+ LIBBPF_OPTS(bpf_test_run_opts, topts);
+ char ctx[8] = {};
+
+ skel->bss->type_id = type_id;
+ topts.ctx_in = ctx;
+ topts.ctx_size_in = sizeof(ctx);
+ if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.dump_type),
+ &topts), "test_run"))
+ return -1;
+ return skel->bss->ret;
+}
+
+void test_snprintf_btf_void(void)
+{
+ const struct btf_type *t;
+ struct snprintf_btf_void *skel;
+ int i, n, cv = 0, var = 0;
+ struct btf *btf;
+
+ btf = btf__parse("/sys/kernel/btf/vmlinux", NULL);
+ if (!btf) {
+ test__skip();
+ return;
+ }
+
+ skel = snprintf_btf_void__open_and_load();
+ if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
+ goto out_btf;
+
+ n = btf__type_cnt(btf);
+ for (i = 1; i < n && !(cv && var); i++) {
+ t = btf__type_by_id(btf, i);
+ if (!cv && btf_kind(t) == BTF_KIND_CONST && t->type == 0)
+ cv = i;
+ /* Pick a VAR small enough to render from the program's buffer. */
+ if (!var && btf_kind(t) == BTF_KIND_VAR) {
+ long sz = btf__resolve_size(btf, t->type);
+
+ if (sz > 0 && sz <= (long)sizeof(skel->bss->obj))
+ var = i;
+ }
+ }
+
+ /* "const void" renders the "<unsupported kind:0>" placeholder. */
+ if (test__start_subtest("const_void")) {
+ if (cv) {
+ ASSERT_EQ(run(skel, cv),
+ sizeof("<unsupported kind:0>") - 1, "ret");
+ ASSERT_STREQ(skel->bss->out, "<unsupported kind:0>",
+ "placeholder");
+ } else {
+ test__skip();
+ }
+ }
+
+ /* A BTF_KIND_VAR must resolve and render without error. */
+ if (test__start_subtest("var")) {
+ if (var)
+ ASSERT_GT(run(skel, var), 0, "ret");
+ else
+ test__skip();
+ }
+
+ snprintf_btf_void__destroy(skel);
+out_btf:
+ btf__free(btf);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/timer_mim.c b/tools/testing/selftests/bpf/prog_tests/timer_mim.c
index c930c7d7105b..fa7bb769ca31 100644
--- a/tools/testing/selftests/bpf/prog_tests/timer_mim.c
+++ b/tools/testing/selftests/bpf/prog_tests/timer_mim.c
@@ -59,10 +59,32 @@ void serial_test_timer_mim(void)
int err;
old_print_fn = libbpf_set_print(NULL);
- timer_reject_skel = timer_mim_reject__open_and_load();
- libbpf_set_print(old_print_fn);
- if (!ASSERT_ERR_PTR(timer_reject_skel, "timer_reject_skel_load"))
+ timer_reject_skel = timer_mim_reject__open();
+ if (!ASSERT_OK_PTR(timer_reject_skel, "timer_reject_skel_open"))
+ goto cleanup;
+ bpf_program__set_autoload(timer_reject_skel->progs.test1, true);
+ err = timer_mim_reject__load(timer_reject_skel);
+ ASSERT_ERR(err, "timer_reject_skel_load");
+ timer_mim_reject__destroy(timer_reject_skel);
+
+ timer_reject_skel = timer_mim_reject__open();
+ if (!ASSERT_OK_PTR(timer_reject_skel, "callback_reject_skel_open"))
goto cleanup;
+ bpf_program__set_autoload(timer_reject_skel->progs.callback_map_uid_mismatch, true);
+ err = timer_mim_reject__load(timer_reject_skel);
+ ASSERT_ERR(err, "callback_reject_skel_load");
+ timer_mim_reject__destroy(timer_reject_skel);
+
+ timer_reject_skel = timer_mim_reject__open();
+ if (!ASSERT_OK_PTR(timer_reject_skel, "callback_accept_skel_open"))
+ goto cleanup;
+ bpf_program__set_autoload(timer_reject_skel->progs.callback_map_uid_match, true);
+ err = timer_mim_reject__load(timer_reject_skel);
+ if (!ASSERT_OK(err, "callback_accept_skel_load"))
+ goto cleanup;
+ timer_mim_reject__destroy(timer_reject_skel);
+ timer_reject_skel = NULL;
+ libbpf_set_print(old_print_fn);
timer_skel = timer_mim__open_and_load();
if (!timer_skel && errno == EOPNOTSUPP) {
@@ -75,6 +97,7 @@ void serial_test_timer_mim(void)
err = timer_mim(timer_skel);
ASSERT_OK(err, "timer_mim");
cleanup:
+ libbpf_set_print(old_print_fn);
timer_mim__destroy(timer_skel);
timer_mim_reject__destroy(timer_reject_skel);
}
diff --git a/tools/testing/selftests/bpf/progs/iters.c b/tools/testing/selftests/bpf/progs/iters.c
index 62d7df9e80be..c6699159dacd 100644
--- a/tools/testing/selftests/bpf/progs/iters.c
+++ b/tools/testing/selftests/bpf/progs/iters.c
@@ -2149,4 +2149,43 @@ __naked int stack_misc_vs_scalar_in_a_loop(void)
);
}
+__used
+static int loop_cb5(int i, __u64 *ctx)
+{
+ /* unsafe on a second iteration */
+ small_arr[*ctx] = i;
+ *ctx = 100500;
+ return 0;
+}
+
+SEC("raw_tp")
+__flag(BPF_F_TEST_STATE_FREQ)
+__failure __msg("memory access is {{.*}} and is outside of the object of size 64")
+__naked void loop_counter_precision_2nd_iter(void)
+{
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "*(u64 *)(r10 - 8) = 0;"
+ "r1 = 2;"
+ "if r0 == 42 goto +1;"
+ "r1 = 1;"
+ "r2 = loop_cb5 ll;"
+ "r3 = r10;"
+ "r3 += -8;"
+ "r4 = 0;"
+ /*
+ * Explore with nr_loops=1 on a first path and nr_loops=2 on a second path.
+ * Buggy verifier did not propagate r1 precision properly,
+ * and thus checkpoints created for nr_loops=1 case matched nr_loops=2 case.
+ */
+ "call %[bpf_loop];"
+ "r0 = 0;"
+ "exit;"
+ :
+ : __imm(bpf_loop),
+ __imm(bpf_get_prandom_u32)
+ : __clobber_all
+ );
+}
+
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/map_kptr_fail.c b/tools/testing/selftests/bpf/progs/map_kptr_fail.c
index 5e25ca806060..eee35d203b66 100644
--- a/tools/testing/selftests/bpf/progs/map_kptr_fail.c
+++ b/tools/testing/selftests/bpf/progs/map_kptr_fail.c
@@ -409,4 +409,41 @@ int reject_scalar_store_to_kptr(struct __sk_buff *ctx)
return 0;
}
+SEC("?tc")
+__description("reject imprecise scalar store to kptr after state pruning")
+__failure __msg("invalid kptr access, R7 type=scalar")
+__naked void reject_imprecise_scalar_store_to_kptr(void)
+{
+ asm volatile (
+ "r0 = 0;"
+ "*(u32 *)(r10 - 4) = r0;"
+ "r2 = r10;"
+ "r2 += -4;"
+ "r1 = %[array_map] ll;"
+ "call %[bpf_map_lookup_elem];"
+ "if r0 == 0 goto l2_%=;"
+ "r6 = r0;"
+ "r9 = *(u64 *)(r6 + 0);"
+ "if r9 != 0 goto l0_%=;"
+ "r7 = 0;"
+ ".rept 10;"
+ "r5 = 1;"
+ ".endr;"
+ "goto l1_%=;"
+ "l0_%=:"
+ "r7 = 0x4141414141414141 ll;"
+ ".rept 10;"
+ "r5 = 1;"
+ ".endr;"
+ "l1_%=:"
+ "*(u64 *)(r6 + 8) = r7;"
+ "l2_%=:"
+ "r0 = 0;"
+ "exit;"
+ :
+ : __imm(bpf_map_lookup_elem),
+ __imm_addr(array_map)
+ : __clobber_all);
+}
+
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c b/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c
index 08379c3b6a03..3701f4ea58c7 100644
--- a/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c
+++ b/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c
@@ -33,6 +33,20 @@ struct {
__type(value, struct elem);
} array SEC(".maps");
+struct kernel_percpu_elem {
+ struct task_struct __percpu_kptr *task;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, int);
+ __type(value, struct kernel_percpu_elem);
+} kernel_percpu_array SEC(".maps");
+
+struct task_struct *bpf_task_from_pid(s32 pid) __ksym;
+void bpf_task_release(struct task_struct *p) __ksym;
+
long ret;
SEC("?fentry/bpf_fentry_test1")
@@ -137,6 +151,51 @@ int BPF_PROG(test_array_map_5)
return 0;
}
+SEC("?syscall")
+__failure __msg("invalid kptr access, R2 type=trusted_ptr_ expected=ptr_task_struct")
+int reject_kernel_ptr_into_percpu_kptr(void *ctx)
+{
+ struct kernel_percpu_elem *e;
+ struct task_struct *p, *old;
+ int index = 0;
+
+ e = bpf_map_lookup_elem(&kernel_percpu_array, &index);
+ if (!e)
+ return 0;
+
+ p = bpf_task_from_pid(1);
+ if (!p)
+ return 0;
+
+ old = bpf_kptr_xchg(&e->task, p);
+ if (old)
+ bpf_task_release(old);
+ return 0;
+}
+
+SEC("?fentry.s/bpf_fentry_test1")
+__failure __msg("invalid kptr access, R2 type=ptr_ expected=ptr_val_t")
+int BPF_PROG(reject_plain_alloc_into_percpu_kptr)
+{
+ struct val_t __percpu_kptr *old;
+ struct val_t *p;
+ struct elem *e;
+ int index = 0;
+
+ e = bpf_map_lookup_elem(&array, &index);
+ if (!e)
+ return 0;
+
+ p = bpf_obj_new(struct val_t);
+ if (!p)
+ return 0;
+
+ old = bpf_kptr_xchg(&e->pc, p);
+ if (old)
+ bpf_percpu_obj_drop(old);
+ return 0;
+}
+
SEC("?fentry.s/bpf_fentry_test1")
__failure __msg("bpf_percpu_obj_new type ID argument must be of a struct of scalars")
int BPF_PROG(test_array_map_6)
diff --git a/tools/testing/selftests/bpf/progs/preempt_lock.c b/tools/testing/selftests/bpf/progs/preempt_lock.c
index 6d5fce7e6ffc..81c459435680 100644
--- a/tools/testing/selftests/bpf/progs/preempt_lock.c
+++ b/tools/testing/selftests/bpf/progs/preempt_lock.c
@@ -115,6 +115,58 @@ int preempt_sleepable_helper(void *ctx)
return 0;
}
+SEC("?uprobe.s")
+__failure __msg("sleepable helper bpf_get_stack#")
+int preempt_sleepable_get_stack(struct pt_regs *ctx)
+{
+ struct bpf_stack_build_id stack;
+
+ bpf_preempt_disable();
+ bpf_get_stack(ctx, &stack, sizeof(stack),
+ BPF_F_USER_STACK | BPF_F_USER_BUILD_ID);
+ bpf_preempt_enable();
+ return 0;
+}
+
+SEC("?uprobe.s")
+__failure __msg("sleepable helper bpf_get_task_stack#")
+int preempt_sleepable_get_task_stack(void *ctx)
+{
+ struct bpf_stack_build_id stack;
+ struct task_struct *task;
+
+ task = bpf_get_current_task_btf();
+ bpf_preempt_disable();
+ bpf_get_task_stack(task, &stack, sizeof(stack),
+ BPF_F_USER_STACK | BPF_F_USER_BUILD_ID);
+ bpf_preempt_enable();
+ return 0;
+}
+
+SEC("?uprobe.s")
+__success
+int sleepable_get_stack(struct pt_regs *ctx)
+{
+ struct bpf_stack_build_id stack;
+
+ bpf_get_stack(ctx, &stack, sizeof(stack),
+ BPF_F_USER_STACK | BPF_F_USER_BUILD_ID);
+ return 0;
+}
+
+SEC("?uprobe.s")
+__success
+int sleepable_get_task_stack(void *ctx)
+{
+ struct bpf_stack_build_id stack;
+ struct task_struct *task;
+
+ task = bpf_get_current_task_btf();
+ bpf_get_task_stack(task, &stack, sizeof(stack),
+ BPF_F_USER_STACK | BPF_F_USER_BUILD_ID);
+ return 0;
+}
+
SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
__failure __msg("kernel func bpf_copy_from_user_str is sleepable within non-preemptible region")
int preempt_sleepable_kfunc(void *ctx)
diff --git a/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c b/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c
index 0d58114a4955..725d73c9ffe1 100644
--- a/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c
+++ b/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c
@@ -22,3 +22,56 @@ int test_raw_tp_null_sched_pi_setprio_arg_2(void *ctx) {
asm volatile("r1 = *(u64 *)(r1 +8); r1 = *(u64 *)(r1 +0);" ::: __clobber_all);
return 0;
}
+
+/* Plain raw tracepoint arguments remain scalar values. */
+SEC("raw_tp/signal_generate")
+__success
+int test_raw_tp_signal_generate_info_scalar(void *ctx)
+{
+ asm volatile("r1 = *(u64 *)(r1 +8); if r1 != 1 goto +0;" ::: __clobber_all);
+ return 0;
+}
+
+/* tp_btf programs may inspect the sentinel as a scalar value. */
+SEC("tp_btf/signal_generate")
+__success
+int test_tp_btf_signal_generate_info_scalar(void *ctx)
+{
+ asm volatile("r1 = *(u64 *)(r1 +8); if r1 != 1 goto +0;" ::: __clobber_all);
+ return 0;
+}
+
+/* SEND_SIG_PRIV is non-NULL, so a NULL check cannot make info safe. */
+SEC("tp_btf/signal_generate")
+__failure __msg("R1 invalid mem access 'scalar'")
+int test_tp_btf_signal_generate_info_no_deref(void *ctx)
+{
+ asm volatile("r1 = *(u64 *)(r1 +8); if r1 == 0 goto +1; "
+ "r1 = *(u32 *)(r1 +0);" ::: __clobber_all);
+ return 0;
+}
+
+SEC("tp_btf/signal_deliver")
+__failure __msg("R1 invalid mem access 'scalar'")
+int test_tp_btf_signal_deliver_info_no_deref(void *ctx)
+{
+ asm volatile("r1 = *(u64 *)(r1 +8); r1 = *(u32 *)(r1 +0);" ::: __clobber_all);
+ return 0;
+}
+
+SEC("tp_btf/sched_process_wait")
+__failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'")
+int test_raw_tp_null_sched_process_wait_arg_1(void *ctx)
+{
+ asm volatile("r1 = *(u64 *)(r1 +0); r1 = *(u32 *)(r1 +0);" ::: __clobber_all);
+ return 0;
+}
+
+SEC("tp_btf/sched_process_wait")
+__success
+int test_raw_tp_null_sched_process_wait_arg_1_checked(void *ctx)
+{
+ asm volatile("r1 = *(u64 *)(r1 +0); if r1 == 0 goto +1; "
+ "r1 = *(u32 *)(r1 +0);" ::: __clobber_all);
+ return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/rbtree_fail.c b/tools/testing/selftests/bpf/progs/rbtree_fail.c
index 555379952dcc..4504608196ab 100644
--- a/tools/testing/selftests/bpf/progs/rbtree_fail.c
+++ b/tools/testing/selftests/bpf/progs/rbtree_fail.c
@@ -16,6 +16,7 @@ struct node_data {
private(A) struct bpf_spin_lock glock;
private(A) struct bpf_rb_root groot __contains(node_data, node);
private(A) struct bpf_rb_root groot2 __contains(node_data, node);
+private(B) struct bpf_res_spin_lock res_glock;
static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b)
{
@@ -265,6 +266,53 @@ static bool less__bad_fn_call_first_unlock_after(struct bpf_rb_node *a, const st
return node_a->key < node_b->key;
}
+static bool less__bad_res_spin_unlock(struct bpf_rb_node *a, const struct bpf_rb_node *b)
+{
+ bpf_res_spin_unlock(&res_glock);
+ return false;
+}
+
+static __noinline void rbtree_cb_unlock_relock(void)
+{
+ bpf_spin_unlock(&glock);
+ bpf_spin_lock(&glock);
+}
+
+static __noinline void rbtree_cb_nested_unlock(void)
+{
+ rbtree_cb_unlock_relock();
+ asm volatile ("");
+}
+
+static bool less__bad_subprog_unlock(struct bpf_rb_node *a, const struct bpf_rb_node *b)
+{
+ struct node_data *node_a;
+ struct node_data *node_b;
+
+ node_a = container_of(a, struct node_data, node);
+ node_b = container_of(b, struct node_data, node);
+ rbtree_cb_nested_unlock();
+
+ return node_a->key < node_b->key;
+}
+
+static __noinline void rbtree_cb_noop(void)
+{
+ asm volatile ("");
+}
+
+static bool less__subprog_allowed(struct bpf_rb_node *a, const struct bpf_rb_node *b)
+{
+ struct node_data *node_a;
+ struct node_data *node_b;
+
+ node_a = container_of(a, struct node_data, node);
+ node_b = container_of(b, struct node_data, node);
+ rbtree_cb_noop();
+
+ return node_a->key < node_b->key;
+}
+
static __always_inline
long add_with_cb(bool (cb)(struct bpf_rb_node *a, const struct bpf_rb_node *b))
{
@@ -301,4 +349,40 @@ long rbtree_api_add_bad_cb_bad_fn_call_first_unlock_after(void *ctx)
return add_with_cb(less__bad_fn_call_first_unlock_after);
}
+SEC("?tc")
+__failure __msg("can't res_spin_{lock,unlock} in rbtree cb")
+long rbtree_api_add_bad_cb_res_spin_unlock(void *ctx)
+{
+ struct node_data *n;
+
+ n = bpf_obj_new(typeof(*n));
+ if (!n)
+ return 1;
+
+ bpf_spin_lock(&glock);
+ if (bpf_res_spin_lock(&res_glock)) {
+ bpf_spin_unlock(&glock);
+ bpf_obj_drop(n);
+ return 1;
+ }
+ bpf_rbtree_add(&groot, &n->node, less__bad_res_spin_unlock);
+ bpf_res_spin_unlock(&res_glock);
+ bpf_spin_unlock(&glock);
+ return 0;
+}
+
+SEC("?tc")
+__failure __msg("can't spin_{lock,unlock} in rbtree cb")
+long rbtree_api_add_bad_cb_subprog_unlock(void *ctx)
+{
+ return add_with_cb(less__bad_subprog_unlock);
+}
+
+SEC("?tc")
+__success
+long rbtree_api_add_cb_subprog_allowed(void *ctx)
+{
+ return add_with_cb(less__subprog_allowed);
+}
+
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/rcu_read_lock.c b/tools/testing/selftests/bpf/progs/rcu_read_lock.c
index 31d4081c3a9f..cdb255addbc3 100644
--- a/tools/testing/selftests/bpf/progs/rcu_read_lock.c
+++ b/tools/testing/selftests/bpf/progs/rcu_read_lock.c
@@ -592,9 +592,9 @@ int non_own_ref_untrusted_ld(void *ctx)
}
bpf_rcu_read_unlock();
/*
- * The unlock leaves node as PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED
- * | NON_OWN_REF, and the load below has to get the BPF_PROBE_MEM
- * rewrite for it, otherwise a bad address panics the kernel.
+ * The unlock leaves node as PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED,
+ * and the load below has to get the BPF_PROBE_MEM rewrite for it,
+ * otherwise a bad address panics the kernel.
*/
non_own_ref_key = node->key;
return 0;
diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr.c b/tools/testing/selftests/bpf/progs/refcounted_kptr.c
index 61906f48025c..cae00f7b0a24 100644
--- a/tools/testing/selftests/bpf/progs/refcounted_kptr.c
+++ b/tools/testing/selftests/bpf/progs/refcounted_kptr.c
@@ -23,6 +23,15 @@ struct map_value {
struct node_data __kptr *node;
};
+struct node_refcount_only {
+ long key;
+ struct bpf_refcount refcount;
+};
+
+struct map_value_refcount_only {
+ struct node_refcount_only __kptr *node;
+};
+
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__type(key, int);
@@ -30,6 +39,13 @@ struct {
__uint(max_entries, 2);
} stashed_nodes SEC(".maps");
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __type(key, int);
+ __type(value, struct map_value_refcount_only);
+ __uint(max_entries, 1);
+} stashed_refcount_only SEC(".maps");
+
struct node_acquire {
long key;
long data;
@@ -832,6 +848,51 @@ long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx)
return 0;
}
+SEC("tc")
+__success
+long refcount_acquire_owning_input_no_null_check(void *ctx)
+{
+ struct node_refcount_only *n, *m;
+
+ n = bpf_obj_new(typeof(*n));
+ if (!n)
+ return 1;
+
+ m = bpf_refcount_acquire(n);
+ bpf_obj_drop(m);
+ bpf_obj_drop(n);
+
+ return 0;
+}
+
+SEC("?syscall")
+__success
+long refcount_acquire_rcu_map_kptr_null_checked(void *ctx)
+{
+ struct map_value_refcount_only *mapval;
+ struct node_refcount_only *n, *m;
+ int idx = 0;
+
+ mapval = bpf_map_lookup_elem(&stashed_refcount_only, &idx);
+ if (!mapval)
+ return 1;
+
+ bpf_rcu_read_lock();
+ n = mapval->node;
+ if (!n) {
+ bpf_rcu_read_unlock();
+ return 2;
+ }
+ m = bpf_refcount_acquire(n);
+ bpf_rcu_read_unlock();
+
+ if (!m)
+ return 3;
+ bpf_obj_drop(m);
+
+ return 0;
+}
+
static long __stash_map_empty_xchg(struct node_data *n, int idx)
{
struct map_value *mapval = bpf_map_lookup_elem(&stashed_nodes, &idx);
diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c
index eaaed0859f94..338e43822ffe 100644
--- a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c
+++ b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c
@@ -19,6 +19,26 @@ struct node_refcounted {
struct bpf_refcount refcount;
};
+struct node_refcount_only {
+ long key;
+ struct bpf_refcount refcount;
+};
+
+struct map_value_refcount_only {
+ struct node_refcount_only __kptr *node;
+};
+
+struct rcu_graph_node {
+ struct bpf_rb_node node;
+ long data;
+};
+
+struct rcu_graph_node *just_here_because_btf_bug;
+
+struct map_value_rcu_graph {
+ struct rcu_graph_node __kptr *node;
+};
+
extern void bpf_rcu_read_lock(void) __ksym;
extern void bpf_rcu_read_unlock(void) __ksym;
@@ -27,6 +47,22 @@ private(A) struct bpf_spin_lock glock;
private(A) struct bpf_rb_root groot __contains(node_acquire, node);
private(B) struct bpf_spin_lock lock;
private(B) struct bpf_list_head head __contains(node_refcounted, list);
+private(C) struct bpf_spin_lock graph_lock;
+private(C) struct bpf_rb_root graph_root __contains(rcu_graph_node, node);
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __type(key, int);
+ __type(value, struct map_value_refcount_only);
+ __uint(max_entries, 1);
+} stashed_refcount_only SEC(".maps");
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __type(key, int);
+ __type(value, struct map_value_rcu_graph);
+ __uint(max_entries, 1);
+} stashed_rcu_graph SEC(".maps");
static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b)
{
@@ -89,6 +125,120 @@ long refcount_acquire_non_object(void *ctx)
return bpf_refcount_acquire(ctx) != NULL;
}
+SEC("?syscall")
+__failure __msg("Possibly NULL pointer passed to trusted R1")
+long refcount_acquire_rcu_map_kptr_unchecked_drop(void *ctx)
+{
+ struct map_value_refcount_only *mapval;
+ struct node_refcount_only *tmp, *n, *m;
+ int idx = 0;
+
+ /* Force Clang to emit complete BTF for struct node_refcount_only. */
+ tmp = bpf_obj_new(typeof(*tmp));
+ if (!tmp)
+ return 3;
+ bpf_obj_drop(tmp);
+
+ mapval = bpf_map_lookup_elem(&stashed_refcount_only, &idx);
+ if (!mapval)
+ return 1;
+
+ bpf_rcu_read_lock();
+ n = mapval->node;
+ if (!n) {
+ bpf_rcu_read_unlock();
+ return 2;
+ }
+ m = bpf_refcount_acquire(n);
+ bpf_rcu_read_unlock();
+
+ bpf_obj_drop(m);
+
+ return 0;
+}
+
+SEC("?syscall")
+__failure
+__msg("bpf_rbtree_remove can only take non-owning or refcounted "
+ "bpf_rb_node pointer")
+long rbtree_remove_after_rcu_unlock(void *ctx)
+{
+ struct map_value_rcu_graph *mapval;
+ struct bpf_rb_node *rb_node;
+ struct rcu_graph_node *node;
+ int idx = 0;
+
+ mapval = bpf_map_lookup_elem(&stashed_rcu_graph, &idx);
+ if (!mapval)
+ return 0;
+
+ bpf_rcu_read_lock();
+ node = mapval->node;
+ if (!node) {
+ bpf_rcu_read_unlock();
+ return 0;
+ }
+ bpf_rcu_read_unlock();
+
+ bpf_spin_lock(&graph_lock);
+ rb_node = bpf_rbtree_remove(&graph_root, &node->node);
+ bpf_spin_unlock(&graph_lock);
+ if (rb_node)
+ bpf_obj_drop(container_of(rb_node, struct rcu_graph_node, node));
+
+ return 0;
+}
+
+SEC("?syscall")
+__failure __msg("R1 is neither owning or non-owning ref")
+long refcount_acquire_after_rcu_unlock(void *ctx)
+{
+ struct map_value_refcount_only *mapval;
+ struct node_refcount_only *node, *ref;
+ int idx = 0;
+
+ mapval = bpf_map_lookup_elem(&stashed_refcount_only, &idx);
+ if (!mapval)
+ return 0;
+
+ bpf_rcu_read_lock();
+ node = mapval->node;
+ if (!node) {
+ bpf_rcu_read_unlock();
+ return 0;
+ }
+ bpf_rcu_read_unlock();
+
+ ref = bpf_refcount_acquire(node);
+ if (ref)
+ bpf_obj_drop(ref);
+
+ return 0;
+}
+
+SEC("?syscall")
+__failure __msg("invalid mem access 'scalar'")
+long graph_kptr_after_spin_unlock(void *ctx)
+{
+ struct map_value_rcu_graph *mapval;
+ struct rcu_graph_node *node;
+ int idx = 0;
+
+ mapval = bpf_map_lookup_elem(&stashed_rcu_graph, &idx);
+ if (!mapval)
+ return 0;
+
+ bpf_spin_lock(&graph_lock);
+ node = mapval->node;
+ if (!node) {
+ bpf_spin_unlock(&graph_lock);
+ return 0;
+ }
+ bpf_spin_unlock(&graph_lock);
+
+ return node->data;
+}
+
SEC("?tc")
__failure __msg("Unreleased reference id=3 alloc_insn={{[0-9]+}}")
long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx)
diff --git a/tools/testing/selftests/bpf/progs/rhash.c b/tools/testing/selftests/bpf/progs/rhash.c
index fc2dac3a719e..aea4de8dc781 100644
--- a/tools/testing/selftests/bpf/progs/rhash.c
+++ b/tools/testing/selftests/bpf/progs/rhash.c
@@ -19,6 +19,11 @@ struct elem {
int val;
};
+struct special_elem {
+ struct task_struct __kptr *task;
+ int val;
+};
+
struct {
__uint(type, BPF_MAP_TYPE_RHASH);
__uint(map_flags, BPF_F_NO_PREALLOC);
@@ -27,6 +32,17 @@ struct {
__type(value, struct elem);
} rhmap SEC(".maps");
+struct {
+ __uint(type, BPF_MAP_TYPE_RHASH);
+ __uint(map_flags, BPF_F_NO_PREALLOC);
+ __uint(max_entries, 1);
+ __type(key, int);
+ __type(value, struct special_elem);
+} special_fields SEC(".maps");
+
+extern struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;
+extern void bpf_task_release(struct task_struct *p) __ksym;
+
SEC("syscall")
int test_rhash_lookup_update(void *ctx)
{
@@ -246,3 +262,99 @@ int test_rhash_delete_nonexistent(void *ctx)
err = 0;
return 0;
}
+
+SEC("syscall")
+int test_rhash_kptr_update(void *ctx)
+{
+ struct special_elem val1 = { .val = 1 };
+ struct special_elem val2 = { .val = 2 };
+ struct task_struct *task, *old;
+ struct special_elem *elem;
+ int key = 0;
+
+ err = 1;
+ if (bpf_map_update_elem(&special_fields, &key, &val1, BPF_NOEXIST))
+ return 1;
+
+ err = 2;
+ elem = bpf_map_lookup_elem(&special_fields, &key);
+ if (!elem)
+ return 2;
+
+ err = 3;
+ task = bpf_task_acquire(bpf_get_current_task_btf());
+ if (!task)
+ return 3;
+
+ err = 4;
+ old = bpf_kptr_xchg(&elem->task, task);
+ if (old) {
+ bpf_task_release(old);
+ return 4;
+ }
+
+ err = 5;
+ if (bpf_map_update_elem(&special_fields, &key, &val2, BPF_EXIST))
+ return 5;
+
+ err = 6;
+ elem = bpf_map_lookup_elem(&special_fields, &key);
+ if (!elem || elem->val != 2)
+ return 6;
+
+ err = 7;
+ old = bpf_kptr_xchg(&elem->task, NULL);
+ if (!old)
+ return 7;
+ bpf_task_release(old);
+
+ err = 8;
+ if (bpf_map_delete_elem(&special_fields, &key))
+ return 8;
+
+ err = 0;
+ return 0;
+}
+
+SEC("syscall")
+int test_rhash_kptr_delete(void *ctx)
+{
+ struct special_elem val = {};
+ struct task_struct *task, *old;
+ struct special_elem *elem;
+ int key = 0;
+
+ err = 1;
+ if (bpf_map_update_elem(&special_fields, &key, &val, BPF_NOEXIST))
+ return 1;
+
+ err = 2;
+ elem = bpf_map_lookup_elem(&special_fields, &key);
+ if (!elem)
+ return 2;
+
+ err = 3;
+ task = bpf_task_acquire(bpf_get_current_task_btf());
+ if (!task)
+ return 3;
+
+ err = 4;
+ old = bpf_kptr_xchg(&elem->task, task);
+ if (old) {
+ bpf_task_release(old);
+ return 4;
+ }
+
+ err = 5;
+ if (bpf_map_delete_elem(&special_fields, &key))
+ return 5;
+
+ err = 6;
+ old = bpf_kptr_xchg(&elem->task, NULL);
+ if (!old)
+ return 6;
+ bpf_task_release(old);
+
+ err = 0;
+ return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/rhash_timer.c b/tools/testing/selftests/bpf/progs/rhash_timer.c
new file mode 100644
index 000000000000..2e06a463c605
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/rhash_timer.c
@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <vmlinux.h>
+#include <errno.h>
+#include <bpf/bpf_helpers.h>
+
+#define CLOCK_MONOTONIC 1
+#define TIMER_NSEC (60ULL * 1000 * 1000 * 1000)
+
+struct timer_value {
+ struct bpf_timer timer;
+ u64 data;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_RHASH);
+ __uint(map_flags, BPF_F_NO_PREALLOC);
+ __uint(max_entries, 1);
+ __type(key, u64);
+ __type(value, struct timer_value);
+} timer_map SEC(".maps");
+
+u64 armed;
+u64 cancelled;
+long timer_init_err;
+long timer_set_callback_err;
+long timer_start_err;
+long timer_cancel_err;
+
+static int timer_cb(void *map, u64 *key, struct timer_value *value)
+{
+ return 0;
+}
+
+static long arm_timer_cb(struct bpf_map *map, u64 *key,
+ struct timer_value *value, void *ctx)
+{
+ u64 key_copy = *key;
+ long err;
+
+ err = bpf_map_delete_elem(map, &key_copy);
+ if (err)
+ return 1;
+
+ err = bpf_timer_init(&value->timer, map, CLOCK_MONOTONIC);
+ if (err) {
+ timer_init_err = err;
+ return 1;
+ }
+
+ err = bpf_timer_set_callback(&value->timer, timer_cb);
+ if (err) {
+ timer_set_callback_err = err;
+ return 1;
+ }
+
+ err = bpf_timer_start(&value->timer, TIMER_NSEC, BPF_F_TIMER_CPU_PIN);
+ if (err) {
+ timer_start_err = err;
+ return 1;
+ }
+
+ __sync_fetch_and_add(&armed, 1);
+ return 1;
+}
+
+static long cancel_timer_cb(struct bpf_map *map, u64 *key,
+ struct timer_value *value, void *ctx)
+{
+ long err;
+
+ err = bpf_timer_cancel(&value->timer);
+ if (err == -EINVAL)
+ return 1;
+ if (err < 0) {
+ timer_cancel_err = err;
+ return 1;
+ }
+
+ __sync_fetch_and_add(&cancelled, 1);
+ return 1;
+}
+
+SEC("syscall")
+int arm_deleted_timer(void *ctx)
+{
+ bpf_for_each_map_elem(&timer_map, arm_timer_cb, NULL, 0);
+ return 0;
+}
+
+SEC("syscall")
+int cancel_recycled_timer(void *ctx)
+{
+ bpf_for_each_map_elem(&timer_map, cancel_timer_cb, NULL, 0);
+ return 0;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/snprintf_btf_void.c b/tools/testing/selftests/bpf/progs/snprintf_btf_void.c
new file mode 100644
index 000000000000..44af80fbbb80
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/snprintf_btf_void.c
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "btf_ptr.h"
+#include <bpf/bpf_helpers.h>
+
+__u32 type_id;
+/* A buffer we own to render the selected type from, kept in bounds. */
+char obj[256];
+char out[64];
+long ret;
+
+SEC("raw_tp/sys_enter")
+int dump_type(void *ctx)
+{
+ struct btf_ptr ptr = {
+ .ptr = obj,
+ .type_id = type_id,
+ .flags = 0,
+ };
+
+ ret = bpf_snprintf_btf(out, sizeof(out), &ptr, sizeof(ptr), 0);
+ return 0;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/tailcall_callback.c b/tools/testing/selftests/bpf/progs/tailcall_callback.c
index c41632cf423b..14fa7a87028e 100644
--- a/tools/testing/selftests/bpf/progs/tailcall_callback.c
+++ b/tools/testing/selftests/bpf/progs/tailcall_callback.c
@@ -45,6 +45,13 @@ int callback_loop(int index, void **cb_ctx)
}
static __noinline
+int callback_tail(int index, void **cb_ctx)
+{
+ bpf_tail_call_static(*cb_ctx, &jmp_table, 0);
+ return 0;
+}
+
+static __noinline
int callback_empty(int index, void *data)
{
return 0;
@@ -78,4 +85,13 @@ int tailcall_callback_2(struct __sk_buff *skb)
return 0;
}
+/* callback with a direct tail call is rejected without a verifier bug */
+SEC("tc")
+__failure __msg("cannot tail call within callback")
+int tailcall_callback_3(struct __sk_buff *skb)
+{
+ bpf_loop(1, callback_tail, &skb, 0);
+ return 0;
+}
+
char __license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/task_local_data.bpf.h b/tools/testing/selftests/bpf/progs/task_local_data.bpf.h
index 0df8a12fd61e..a31a399870be 100644
--- a/tools/testing/selftests/bpf/progs/task_local_data.bpf.h
+++ b/tools/testing/selftests/bpf/progs/task_local_data.bpf.h
@@ -61,6 +61,7 @@
#define TLD_ROUND_UP(x, y) ((((x) - 1) | TLD_ROUND_MASK(x, y)) + 1)
#define TLD_MAX_DATA_CNT (__PAGE_SIZE / sizeof(struct tld_metadata) - 1)
+#define TLD_DATA_SIZE (__PAGE_SIZE - sizeof(__u64))
#ifndef TLD_NAME_LEN
#define TLD_NAME_LEN 62
@@ -189,6 +190,8 @@ static int __tld_fetch_key(struct tld_object *tld_obj, const char *name, int i_s
return start + off;
off += TLD_ROUND_UP(metadata[i].size, 8);
+ if (off > TLD_DATA_SIZE)
+ break;
}
return -cnt;
diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf.c b/tools/testing/selftests/bpf/progs/test_bpf_nf.c
index df43649ecb78..eda9b7bbab75 100644
--- a/tools/testing/selftests/bpf/progs/test_bpf_nf.c
+++ b/tools/testing/selftests/bpf/progs/test_bpf_nf.c
@@ -190,8 +190,8 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32,
ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def,
sizeof(opts_def));
if (ct) {
- __u16 sport = bpf_get_prandom_u32();
- __u16 dport = bpf_get_prandom_u32();
+ __u16 sport = bpf_get_prandom_u32() % 65535 + 1;
+ __u16 dport = bpf_get_prandom_u32() % 65535 + 1;
union nf_inet_addr saddr = {};
union nf_inet_addr daddr = {};
struct nf_conn *ct_ins;
@@ -293,8 +293,8 @@ nf_ct_opts_new_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *
ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def,
sizeof(opts_def));
if (ct) {
- __u16 sport = bpf_get_prandom_u32();
- __u16 dport = bpf_get_prandom_u32();
+ __u16 sport = bpf_get_prandom_u32() % 65535 + 1;
+ __u16 dport = bpf_get_prandom_u32() % 65535 + 1;
union nf_inet_addr saddr = {};
union nf_inet_addr daddr = {};
struct nf_conn *ct_ins;
diff --git a/tools/testing/selftests/bpf/progs/timer_mim_reject.c b/tools/testing/selftests/bpf/progs/timer_mim_reject.c
index dd3f1ed6d6e6..83f31138336b 100644
--- a/tools/testing/selftests/bpf/progs/timer_mim_reject.c
+++ b/tools/testing/selftests/bpf/progs/timer_mim_reject.c
@@ -43,7 +43,7 @@ static int timer_cb(void *map, int *key, struct hmap_elem *val)
return 0;
}
-SEC("fentry/bpf_fentry_test1")
+SEC("?fentry/bpf_fentry_test1")
int BPF_PROG(test1, int a)
{
struct hmap_elem init = {};
@@ -72,3 +72,85 @@ int BPF_PROG(test1, int a)
err |= 8;
return 0;
}
+
+struct callback_ctx {
+ void *map;
+};
+
+static int mismatch_iter_cb(void *map, int *key, struct hmap_elem *val, struct callback_ctx *ctx)
+{
+ bpf_timer_init(&val->timer, ctx->map, CLOCK_MONOTONIC);
+ return 0;
+}
+
+static int timer_mismatch_cb(void *map, int *key, struct hmap_elem *val)
+{
+ struct callback_ctx ctx = { .map = map };
+ struct bpf_map *inner_map2;
+ int array_key2 = ARRAY_KEY2;
+
+ inner_map2 = bpf_map_lookup_elem(&outer_arr, &array_key2);
+ if (!inner_map2)
+ return 0;
+ bpf_for_each_map_elem(inner_map2, mismatch_iter_cb, &ctx, 0);
+ return 0;
+}
+
+static int match_iter_cb(void *map, int *key, struct hmap_elem *val, struct callback_ctx *ctx)
+{
+ bpf_timer_init(&val->timer, map, CLOCK_MONOTONIC);
+ return 0;
+}
+
+static int timer_match_cb(void *map, int *key, struct hmap_elem *val)
+{
+ struct callback_ctx ctx = {};
+ struct bpf_map *inner_map2;
+ int array_key2 = ARRAY_KEY2;
+
+ inner_map2 = bpf_map_lookup_elem(&outer_arr, &array_key2);
+ if (!inner_map2)
+ return 0;
+ bpf_for_each_map_elem(inner_map2, match_iter_cb, &ctx, 0);
+ return 0;
+}
+
+SEC("?fentry/bpf_fentry_test1")
+int BPF_PROG(callback_map_uid_mismatch, int a)
+{
+ struct hmap_elem *val;
+ struct bpf_map *inner_map;
+ int array_key = ARRAY_KEY;
+ int hash_key = HASH_KEY;
+
+ inner_map = bpf_map_lookup_elem(&outer_arr, &array_key);
+ if (!inner_map)
+ return 0;
+ val = bpf_map_lookup_elem(inner_map, &hash_key);
+ if (!val)
+ return 0;
+
+ bpf_timer_init(&val->timer, inner_map, CLOCK_MONOTONIC);
+ bpf_timer_set_callback(&val->timer, timer_mismatch_cb);
+ return 0;
+}
+
+SEC("?fentry/bpf_fentry_test1")
+int BPF_PROG(callback_map_uid_match, int a)
+{
+ struct hmap_elem *val;
+ struct bpf_map *inner_map;
+ int array_key = ARRAY_KEY;
+ int hash_key = HASH_KEY;
+
+ inner_map = bpf_map_lookup_elem(&outer_arr, &array_key);
+ if (!inner_map)
+ return 0;
+ val = bpf_map_lookup_elem(inner_map, &hash_key);
+ if (!val)
+ return 0;
+
+ bpf_timer_init(&val->timer, inner_map, CLOCK_MONOTONIC);
+ bpf_timer_set_callback(&val->timer, timer_match_cb);
+ return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c b/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c
index 6bf95550a024..e0926767bbd3 100644
--- a/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c
+++ b/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c
@@ -62,6 +62,104 @@ int timer_sleepable_prog(void *ctx)
return 0;
}
+static int timer_sys_bpf_cb(void *map, int *key, struct bpf_timer *timer)
+{
+ __u64 attr = 0;
+
+ bpf_sys_bpf(BPF_MAP_FREEZE, &attr, sizeof(attr));
+ return 0;
+}
+
+SEC("syscall")
+__failure __msg("sleepable helper bpf_sys_bpf#{{[0-9]+}} in non-sleepable prog")
+int timer_sys_bpf_prog(void *ctx)
+{
+ struct timer_elem *val;
+ int key = 0;
+
+ val = bpf_map_lookup_elem(&timer_map, &key);
+ if (!val)
+ return 0;
+
+ bpf_timer_init(&val->t, &timer_map, 0);
+ bpf_timer_set_callback(&val->t, timer_sys_bpf_cb);
+ return 0;
+}
+
+static int timer_sys_close_cb(void *map, int *key, struct bpf_timer *timer)
+{
+ bpf_sys_close(0);
+ return 0;
+}
+
+SEC("syscall")
+__failure __msg("sleepable helper bpf_sys_close#{{[0-9]+}} in non-sleepable prog")
+int timer_sys_close_prog(void *ctx)
+{
+ struct timer_elem *val;
+ int key = 0;
+
+ val = bpf_map_lookup_elem(&timer_map, &key);
+ if (!val)
+ return 0;
+
+ bpf_timer_init(&val->t, &timer_map, 0);
+ bpf_timer_set_callback(&val->t, timer_sys_close_cb);
+ return 0;
+}
+
+static int timer_btf_find_cb(void *map, int *key, struct bpf_timer *timer)
+{
+ char name[] = "task_struct";
+
+ bpf_btf_find_by_name_kind(name, sizeof(name), BTF_KIND_STRUCT, 0);
+ return 0;
+}
+
+SEC("syscall")
+__failure __msg("sleepable helper bpf_btf_find_by_name_kind#{{[0-9]+}} in non-sleepable prog")
+int timer_btf_find_prog(void *ctx)
+{
+ struct timer_elem *val;
+ int key = 0;
+
+ val = bpf_map_lookup_elem(&timer_map, &key);
+ if (!val)
+ return 0;
+
+ bpf_timer_init(&val->t, &timer_map, 0);
+ bpf_timer_set_callback(&val->t, timer_btf_find_cb);
+ return 0;
+}
+
+SEC("syscall")
+__success
+int syscall_sys_bpf_prog(void *ctx)
+{
+ __u64 attr = 0;
+
+ bpf_sys_bpf(BPF_MAP_FREEZE, &attr, sizeof(attr));
+ return 0;
+}
+
+SEC("syscall")
+__success
+int syscall_sys_close_prog(void *ctx)
+{
+ bpf_sys_close(0);
+ return 0;
+}
+
+SEC("syscall")
+__success
+int syscall_btf_find_prog(void *ctx)
+{
+ char name[] = "task_struct";
+
+ bpf_btf_find_by_name_kind(name, sizeof(name), BTF_KIND_STRUCT, 0);
+ return 0;
+}
+
/* Workqueue tests */
struct wq_elem {
diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c
index 1a273e416fed..df8d5309657e 100644
--- a/tools/testing/selftests/bpf/progs/verifier_bounds.c
+++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c
@@ -2267,6 +2267,47 @@ __naked void deduce64_from_32_wrapping_32bit(void)
: __clobber_all);
}
+/*
+ * Unprivileged variable pointer arithmetic on a PTR_TO_MAP_VALUE whose
+ * offset collapses to a constant. The Spectre-v1 speculative path snapshots
+ * the pointer while its r32 has just been blanked but its offset not yet
+ * synced; the following register move used to trip reg_bounds_sanity_check()
+ * ("const subreg tnum out of sync with range bounds"). With
+ * BPF_F_TEST_REG_INVARIANTS that violation turns into a load failure, so the
+ * unprivileged program must still load.
+ */
+SEC("socket")
+__success __success_unpriv
+__flag(BPF_F_TEST_REG_INVARIANTS)
+__naked void spec_ptr_alu_const_offset(void)
+{
+ asm volatile (" \
+ call %[bpf_ktime_get_ns]; \
+ *(u64*)(r10 - 16) = r0; \
+ r1 = 0; \
+ *(u64*)(r10 - 8) = r1; \
+ r2 = r10; \
+ r2 += -8; \
+ r1 = %[map_hash_8b] ll; \
+ call %[bpf_map_lookup_elem]; \
+ if r0 == 0 goto l0_%=; \
+ r1 = *(u64*)(r10 - 16); \
+ r2 = 0x40000000; \
+ if r1 > r2 goto l0_%=; \
+ if r1 s> 1 goto l0_%=; /* r1 in [0, 1] */ \
+ r0 += r1; /* ptr += bounded scalar */ \
+ r9 = r0; /* used to trip the warning */ \
+ *(u8*)(r0 + 0) = r1; \
+l0_%=: r0 = 0; \
+ exit; \
+ "
+ :
+ : __imm(bpf_ktime_get_ns),
+ __imm(bpf_map_lookup_elem),
+ __imm_addr(map_hash_8b)
+ : __clobber_all);
+}
+
/* Check that range_within() compares cnum ranges, not min/max projections. */
SEC("socket")
__failure __msg("div by zero")
diff --git a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c
index 328cf630210a..a73b837553fb 100644
--- a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c
+++ b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c
@@ -621,6 +621,116 @@ __naked void helper_call_does_not_prevent_bpf_fastcall(void)
: __clobber_all);
}
+/* A jump to the first spill executes the whole pattern, rewrite is safe. */
+SEC("raw_tp")
+__arch_x86_64
+__log_level(4)
+__msg("subprog 0 (jump_to_first_spill) main {{.*}} stack 0")
+__xlated("2: if r0 == 0x2a goto pc+0")
+__xlated("3: r0 = ")
+__xlated("4: r0 = &(void __percpu *)(r0)")
+__success
+__naked void jump_to_first_spill(void)
+{
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "r1 = 1;"
+ "if r0 == 42 goto l0_%=;"
+"l0_%=:"
+ "*(u64 *)(r10 - 8) = r1;"
+ "call %[bpf_get_smp_processor_id];"
+ "r1 = *(u64 *)(r10 - 8);"
+ "exit;"
+ :
+ : __imm(bpf_get_prandom_u32),
+ __imm(bpf_get_smp_processor_id)
+ : __clobber_all);
+}
+
+/* A jump to the call skips the spill, the pattern must be kept. */
+SEC("raw_tp")
+__arch_x86_64
+__log_level(4)
+__msg("subprog 0 (jump_to_call) main {{.*}} stack 8")
+__xlated("2: if r0 == 0x2a goto pc+1")
+__xlated("3: *(u64 *)(r10 -8) = r1")
+__xlated("...")
+__xlated("7: r1 = *(u64 *)(r10 -8)")
+__success
+__naked void jump_to_call(void)
+{
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "r1 = 1;"
+ "if r0 == 42 goto l0_%=;"
+ "*(u64 *)(r10 - 8) = r1;"
+"l0_%=:"
+ "call %[bpf_get_smp_processor_id];"
+ "r1 = *(u64 *)(r10 - 8);"
+ "exit;"
+ :
+ : __imm(bpf_get_prandom_u32),
+ __imm(bpf_get_smp_processor_id)
+ : __clobber_all);
+}
+
+/* A jump to the fill skips the spill, the pattern must be kept. */
+SEC("raw_tp")
+__arch_x86_64
+__log_level(4)
+__msg("subprog 0 (jump_to_fill) main {{.*}} stack 8")
+__xlated("2: if r0 == 0x2a goto pc+4")
+__xlated("3: *(u64 *)(r10 -8) = r1")
+__xlated("...")
+__xlated("7: r1 = *(u64 *)(r10 -8)")
+__success
+__naked void jump_to_fill(void)
+{
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "r1 = 1;"
+ "if r0 == 42 goto l0_%=;"
+ "*(u64 *)(r10 - 8) = r1;"
+ "call %[bpf_get_smp_processor_id];"
+"l0_%=:"
+ "r1 = *(u64 *)(r10 - 8);"
+ "exit;"
+ :
+ : __imm(bpf_get_prandom_u32),
+ __imm(bpf_get_smp_processor_id)
+ : __clobber_all);
+}
+
+/* Same as above, but the fill is entered by an unconditional jump. */
+SEC("raw_tp")
+__arch_x86_64
+__log_level(4)
+__msg("subprog 0 (unconditional_jump_to_fill) main {{.*}} stack 8")
+__xlated("3: *(u64 *)(r10 -8) = r1")
+__xlated("...")
+__xlated("7: r1 = *(u64 *)(r10 -8)")
+__xlated("8: exit")
+__xlated("9: goto pc-3")
+__success
+__naked void unconditional_jump_to_fill(void)
+{
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "r1 = 1;"
+ "if r0 == 42 goto l1_%=;"
+ "*(u64 *)(r10 - 8) = r1;"
+ "call %[bpf_get_smp_processor_id];"
+"l0_%=:"
+ "r1 = *(u64 *)(r10 - 8);"
+ "exit;"
+"l1_%=:"
+ "goto l0_%=;"
+ :
+ : __imm(bpf_get_prandom_u32),
+ __imm(bpf_get_smp_processor_id)
+ : __clobber_all);
+}
+
SEC("raw_tp")
__arch_x86_64
__log_level(4)
diff --git a/tools/testing/selftests/bpf/progs/verifier_cfg.c b/tools/testing/selftests/bpf/progs/verifier_cfg.c
index c1f55e1d80a4..3c3bb03e8217 100644
--- a/tools/testing/selftests/bpf/progs/verifier_cfg.c
+++ b/tools/testing/selftests/bpf/progs/verifier_cfg.c
@@ -3,6 +3,7 @@
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
+#include "../../../include/linux/filter.h"
#include "bpf_misc.h"
SEC("socket")
@@ -56,6 +57,19 @@ __naked void out_of_range_jump2(void)
}
SEC("socket")
+__description("invalid DW LDSX instruction in diagnostics")
+__failure __msg("BUG_ldx_99")
+__log_level(2)
+__naked void invalid_dw_ldsx(void)
+{
+ asm volatile (" \
+ .8byte %[ldsx_dw]; \
+" :
+ : __imm_insn(ldsx_dw, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | BPF_DW, BPF_REG_0, BPF_REG_0, 0, 0))
+ : __clobber_all);
+}
+
+SEC("socket")
__description("loop (back-edge)")
__failure __msg("unreachable insn 1")
__msg_unpriv("back-edge")
diff --git a/tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c b/tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c
index 9a13f5c11ac7..884080a5bffc 100644
--- a/tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c
+++ b/tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c
@@ -305,4 +305,33 @@ __naked void cpu_cgroup_storage_access_6(void)
: __clobber_all);
}
+/*
+ * Verification takes two paths: with r2 being scalar zero on path (1)
+ * and with r2 being some other scalar on path (2).
+ * Check that the verifier does not use checkpoints created
+ * on path (1) to prune path (2).
+ */
+SEC("cgroup/skb")
+__failure
+__flag(BPF_F_TEST_STATE_FREQ)
+__msg("get_local_storage() doesn't support non-zero flags")
+__naked void non_zero_flags_on_a_pruned_path(void)
+{
+ asm volatile (" \
+ call %[bpf_get_prandom_u32]; \
+ /* r2 is 0 on the path explored first, 1 on the other */\
+ r2 = 1; \
+ if r0 == 0 goto 1f; \
+ r2 = 0; \
+1: r1 = %[cgroup_storage] ll; \
+ call %[bpf_get_local_storage]; \
+ r0 = 0; \
+ exit; \
+" :
+ : __imm(bpf_get_prandom_u32),
+ __imm(bpf_get_local_storage),
+ __imm_addr(cgroup_storage)
+ : __clobber_all);
+}
+
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c
index 0bdeb7bc4687..a3d2af8dc839 100644
--- a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c
+++ b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c
@@ -56,6 +56,30 @@ int trusted_task_arg_nullable(void *ctx)
return res;
}
+/*
+ * Check that the verifier does not use checkpoints created
+ * on path with r1 == 0 to prune path with r1 != 0.
+ */
+SEC("?tp_btf/task_newtask")
+__failure
+__flag(BPF_F_TEST_STATE_FREQ)
+__msg("R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_")
+__naked int null_btf_id_arg_global_subprog(void)
+{
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "r1 = 42;"
+ "if r0 > 42 goto 1f;"
+ "r1 = 0;"
+ "1:"
+ "call subprog_trusted_task_nullable;"
+ "r0 = 0;"
+ "exit;"
+ :
+ : __imm(bpf_get_prandom_u32)
+ : __clobber_all);
+}
+
__weak int subprog_trusted_task_nonnull(struct task_struct *task __arg_trusted)
{
return task->pid + task->tgid;
diff --git a/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c b/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c
index 75dd922e4e9f..1fbcc5228306 100644
--- a/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c
+++ b/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c
@@ -169,6 +169,23 @@ static int iter_limit_cb(__u32 idx, struct num_context *ctx)
}
SEC("?raw_tp")
+__failure __msg("R1 type=ctx expected=scalar")
+__naked void bpf_loop_reject_pointer(void)
+{
+ asm volatile (
+ "r2 = %[iter_limit_cb];"
+ "r3 = 0;"
+ "r4 = 0;"
+ "call %[bpf_loop];"
+ "exit;"
+ :
+ : __imm_ptr(iter_limit_cb),
+ __imm(bpf_loop)
+ : __clobber_common
+ );
+}
+
+SEC("?raw_tp")
__success
int bpf_loop_iter_limit_ok(void *unused)
{
diff --git a/tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c b/tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c
index 3d1e8de4390c..3c789c565b18 100644
--- a/tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c
+++ b/tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c
@@ -3,7 +3,9 @@
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
+#include <stdbool.h>
#include "bpf_misc.h"
+#include "bpf_kfuncs.h"
struct {
__uint(type, BPF_MAP_TYPE_XSKMAP);
@@ -12,6 +14,13 @@ struct {
__type(value, int);
} map_xskmap SEC(".maps");
+struct {
+ __uint(type, BPF_MAP_TYPE_HASH);
+ __uint(max_entries, 1);
+ __type(key, int);
+ __type(value, int);
+} map_hash SEC(".maps");
+
/* This is equivalent to the following program:
*
* r6 = skb->sk;
@@ -264,4 +273,188 @@ __naked void jne_reg_reg_null_check(void)
: __clobber_all);
}
+/*
+ * A comparison between PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED and
+ * PTR_TO_MAP_VALUE_OR_NULL should not infer that map pointer is not null.
+ * A bug in check_cond_jmp_op() made such inference possible.
+ */
+SEC("raw_tp")
+__failure
+__msg("error: invalid dereference of R0 (a nullable map value pointer)")
+__msg(">>> 11 | (61) r0 = *(u32 *)(r0 +0)")
+__naked void untrusted_mem_does_not_infer_map_value_non_null(void)
+{
+ asm volatile (" \
+ /* r6 = bpf_rdonly_cast(0, 0); */ \
+ r1 = 0; \
+ r2 = 0; \
+ call %[bpf_rdonly_cast]; \
+ r6 = r0; \
+ /* r0 = bpf_map_lookup_elem(map_hash, &key); */ \
+ *(u64 *)(r10 - 8) = 0; \
+ r1 = %[map_hash] ll; \
+ r2 = r10; \
+ r2 += -8; \
+ call %[bpf_map_lookup_elem]; \
+ /* \
+ * buggy verifier assumed that r6 can't be null \
+ * and marked r0 non-null as well. \
+ */ \
+ if r6 != r0 goto 1f; \
+ r0 = *(u32 *)(r0 + 0); \
+1: r0 = 0; \
+ exit; \
+" :
+ : __imm(bpf_rdonly_cast),
+ __imm(bpf_map_lookup_elem),
+ __imm_addr(map_hash)
+ : __clobber_all);
+}
+
+/*
+ * A pointer with an offset that is not bounded from above may be null at
+ * runtime, hence it is not a witness for the pointer it is compared with.
+ */
+SEC("socket")
+__failure
+__msg("error: invalid dereference of R7 (a nullable map value pointer)")
+__naked void unbounded_offset_does_not_infer_map_value_non_null(void)
+{
+ asm volatile (" \
+ /* r6 = bpf_map_lookup_elem(map_hash, &0); */ \
+ *(u64 *)(r10 - 8) = 0; \
+ r1 = %[map_hash] ll; \
+ r2 = r10; \
+ r2 += -8; \
+ call %[bpf_map_lookup_elem]; \
+ if r0 == 0 goto 1f; \
+ r6 = r0; \
+ /* r7 = bpf_map_lookup_elem(map_hash, &1); */ \
+ *(u64 *)(r10 - 8) = 1; \
+ r1 = %[map_hash] ll; \
+ r2 = r10; \
+ r2 += -8; \
+ call %[bpf_map_lookup_elem]; \
+ r7 = r0; \
+ /* pointer - pointer is an unknown scalar */ \
+ r8 = r7; \
+ r8 -= r6; \
+ /* r8 is in [0, S64_MAX] */ \
+ r8 <<= 1; \
+ r8 >>= 1; \
+ /* r6 may wrap to zero at runtime */ \
+ r6 += r8; \
+ if r7 != r6 goto 1f; \
+ r0 = *(u8 *)(r7 + 0); \
+1: r0 = 0; \
+ exit; \
+" :
+ : __imm(bpf_map_lookup_elem),
+ __imm_addr(map_hash)
+ : __clobber_all);
+}
+
+/* Same, but the offset is bounded, so the inference is still done. */
+SEC("socket")
+__success
+__naked void bounded_offset_infers_map_value_non_null(void)
+{
+ asm volatile (" \
+ /* r6 = bpf_map_lookup_elem(map_hash, &0); */ \
+ *(u64 *)(r10 - 8) = 0; \
+ r1 = %[map_hash] ll; \
+ r2 = r10; \
+ r2 += -8; \
+ call %[bpf_map_lookup_elem]; \
+ if r0 == 0 goto 1f; \
+ r6 = r0; \
+ /* r7 = bpf_map_lookup_elem(map_hash, &1); */ \
+ *(u64 *)(r10 - 8) = 1; \
+ r1 = %[map_hash] ll; \
+ r2 = r10; \
+ r2 += -8; \
+ call %[bpf_map_lookup_elem]; \
+ r7 = r0; \
+ /* pointer - pointer is an unknown scalar */ \
+ r8 = r7; \
+ r8 -= r6; \
+ /* r8 is in [0, 3] */ \
+ r8 &= 3; \
+ r6 += r8; \
+ if r7 != r6 goto 1f; \
+ r0 = *(u8 *)(r7 + 0); \
+1: r0 = 0; \
+ exit; \
+" :
+ : __imm(bpf_map_lookup_elem),
+ __imm_addr(map_hash)
+ : __clobber_all);
+}
+
+/*
+ * The low 32 bits of a map value pointer may be zero, hence a 32-bit
+ * compare with zero cannot be predicted from the pointer being non-NULL
+ * and both successors of such a jump have to be verified.
+ */
+SEC("socket")
+__failure __msg("invalid access to map value, value_size=4 off=32 size=4")
+__naked void jmp32_ptr_vs_zero_jne(void)
+{
+ asm volatile (" \
+ /* r0 = bpf_map_lookup_elem(map_hash, &key); */ \
+ *(u64 *)(r10 - 8) = 0; \
+ r1 = %[map_hash] ll; \
+ r2 = r10; \
+ r2 += -8; \
+ call %[bpf_map_lookup_elem]; \
+ if r0 == 0 goto 1f; \
+ if w0 != 0 goto 1f; \
+ r0 = *(u32 *)(r0 + 32); \
+1: r0 = 0; \
+ exit; \
+" :
+ : __imm(bpf_map_lookup_elem),
+ __imm_addr(map_hash)
+ : __clobber_all);
+}
+
+/*
+ * The below program is explored in two paths: r6 == 0 and r6 == 1.
+ * On the first path comparison "if r0 == r6 goto 2f" should mark r6 as precise,
+ * otherwise unsafe path with r6 == 1 would be incorrectly pruned.
+ */
+SEC("socket")
+__failure
+__flag(BPF_F_TEST_STATE_FREQ)
+__msg("error: invalid dereference of R0 (a nullable map value pointer)")
+__naked void imprecise_zero_does_not_infer_map_value_non_null(void)
+{
+ asm volatile (" \
+ call %[bpf_get_prandom_u32]; \
+ /* r6 is 0 on the path explored first, 1 on the other */\
+ r6 = 1; \
+ if r0 == 0 goto 1f; \
+ r6 = 0; \
+ /* r0 = bpf_map_lookup_elem(map_hash, &0); */ \
+1: *(u64 *)(r10 - 8) = 0; \
+ r1 = %[map_hash] ll; \
+ r2 = r10; \
+ r2 += -8; \
+ call %[bpf_map_lookup_elem]; \
+ if r0 == r6 goto 2f; \
+ r0 = *(u8 *)(r0 + 0); \
+2: r0 = 0; \
+ exit; \
+" :
+ : __imm(bpf_get_prandom_u32),
+ __imm(bpf_map_lookup_elem),
+ __imm_addr(map_hash)
+ : __clobber_all);
+}
+
+void kfunc_root(void)
+{
+ bpf_rdonly_cast(0, 0);
+}
+
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_ld_ind.c b/tools/testing/selftests/bpf/progs/verifier_ld_ind.c
index 09e81b99eecb..32989f981fb6 100644
--- a/tools/testing/selftests/bpf/progs/verifier_ld_ind.c
+++ b/tools/testing/selftests/bpf/progs/verifier_ld_ind.c
@@ -194,6 +194,102 @@ __naked void ld_ind_subprog_both_paths_safe(void)
::: __clobber_all);
}
+__naked __noinline __used
+static int ld_abs_callback(void)
+{
+ asm volatile (
+ "r6 = *(u64 *)(r2 + 0);"
+ ".8byte %[ld_abs];"
+ "r0 = 0;"
+ "exit;"
+ :
+ : __imm_insn(ld_abs, BPF_LD_ABS(BPF_W, 0))
+ : __clobber_all);
+}
+
+SEC("socket")
+__description("ld_abs: reject in callback")
+__failure __msg("cannot use BPF_LD_[ABS|IND] within callback")
+int ld_abs_callback_reject(struct __sk_buff *skb)
+{
+ bpf_loop(1, ld_abs_callback, &skb, 0);
+ return 0;
+}
+
+__naked __noinline __used
+static int ld_ind_callback_subprog(void)
+{
+ asm volatile (
+ "r6 = r1;"
+ "r7 = 0;"
+ ".8byte %[ld_ind];"
+ "r0 = 0;"
+ "exit;"
+ :
+ : __imm_insn(ld_ind, BPF_LD_IND(BPF_W, BPF_REG_7, 0))
+ : __clobber_all);
+}
+
+__naked __noinline __used
+static int ld_ind_callback(void)
+{
+ asm volatile (
+ "r1 = *(u64 *)(r2 + 0);"
+ "call ld_ind_callback_subprog;"
+ "exit;"
+ ::: __clobber_all);
+}
+
+SEC("socket")
+__description("ld_ind: reject in callback subprog")
+__failure __msg("cannot use BPF_LD_[ABS|IND] within callback")
+int ld_ind_callback_subprog_reject(struct __sk_buff *skb)
+{
+ bpf_loop(1, ld_ind_callback, &skb, 0);
+ return 0;
+}
+
+static __noinline int ld_ind_global_static(struct __sk_buff *skb)
+{
+ asm volatile (
+ "r6 = %[skb];"
+ "r7 = 0;"
+ ".8byte %[ld_ind];"
+ :
+ : [skb] "r"(skb),
+ __imm_insn(ld_ind, BPF_LD_IND(BPF_W, BPF_REG_7, 0))
+ : __clobber_common, "r6", "r7");
+ return skb->mark;
+}
+
+__noinline int ld_ind_global(struct __sk_buff *skb)
+{
+ return ld_ind_global_static(skb);
+}
+
+static int ld_ind_global_callback(__u32 index, struct __sk_buff **ctx)
+{
+ ld_ind_global(*ctx);
+ return 0;
+}
+
+SEC("socket")
+__description("ld_ind: reject in callback global subprog")
+__failure __msg("cannot use BPF_LD_[ABS|IND] within callback")
+int ld_ind_global_callback_reject(struct __sk_buff *skb)
+{
+ bpf_loop(1, ld_ind_global_callback, &skb, 0);
+ return 0;
+}
+
+SEC("socket")
+__description("ld_ind: allow in non-callback global subprog")
+__success
+int ld_ind_global_subprog_ok(struct __sk_buff *skb)
+{
+ return ld_ind_global(skb);
+}
+
/*
* ld_{abs,ind} in subprogs require scalar (int) return type in BTF.
* A test with void return must be rejected.
diff --git a/tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c b/tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c
index e2cbc5bda65e..b5d7f567d0d4 100644
--- a/tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c
+++ b/tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c
@@ -113,4 +113,82 @@ int with_valid_ctx_access_test6(struct bpf_nf_ctx *ctx)
return th->dest == bpf_htons(22) ? NF_ACCEPT : NF_DROP;
}
+SEC("netfilter")
+__description("netfilter test prog with skb write access")
+__failure __msg("only read is supported")
+int skb_len_write(struct bpf_nf_ctx *ctx)
+{
+ ctx->skb->len = 1;
+ return 1;
+}
+
+SEC("netfilter")
+__description("netfilter test prog with skb data write access")
+__failure __msg("cannot write into rdonly_untrusted_mem")
+int skb_data_write(struct bpf_nf_ctx *ctx)
+{
+ ctx->skb->data[0] = 0;
+ return 1;
+}
+
+SEC("netfilter")
+__description("netfilter test prog with bpf_dynptr_write")
+__success __failure_unpriv
+__retval(0)
+int with_dynptr_write(struct bpf_nf_ctx *ctx)
+{
+ struct __sk_buff *skb = (struct __sk_buff *)ctx->skb;
+ struct bpf_dynptr ptr;
+ u8 buffer[1] = {};
+
+ if (bpf_dynptr_from_skb(skb, 0, &ptr))
+ return 1;
+
+ if (bpf_dynptr_write(&ptr, 0, buffer, sizeof(buffer), 0))
+ return 0; /* must always fail */
+
+ return 1;
+}
+
+SEC("netfilter")
+__description("netfilter test prog with bpf_dynptr_slice_rdwr")
+__failure __msg("the prog does not allow writes to packet data")
+int with_dynptr_rdwr(struct bpf_nf_ctx *ctx)
+{
+ struct __sk_buff *skb = (struct __sk_buff *)ctx->skb;
+ u8 buffer_iph[20] = {};
+ struct bpf_dynptr ptr;
+ struct iphdr *iph;
+
+ if (bpf_dynptr_from_skb(skb, 0, &ptr))
+ return 1;
+
+ iph = bpf_dynptr_slice_rdwr(&ptr, 0, buffer_iph, sizeof(buffer_iph));
+ if (!iph)
+ return 0;
+
+ return 1;
+}
+
+SEC("netfilter")
+__description("netfilter test prog with bpf_dynptr_slice + write")
+__failure __msg("cannot write into rdonly_mem")
+int with_dynptr_store(struct bpf_nf_ctx *ctx)
+{
+ struct __sk_buff *skb = (struct __sk_buff *)ctx->skb;
+ u8 buffer_iph[20] = {};
+ struct bpf_dynptr ptr;
+ struct iphdr *iph;
+
+ if (bpf_dynptr_from_skb(skb, 0, &ptr))
+ return 1;
+
+ iph = bpf_dynptr_slice(&ptr, 0, buffer_iph, sizeof(buffer_iph));
+ if (!iph)
+ return 0;
+ iph->protocol = 42;
+
+ return 1;
+}
+
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_precision.c b/tools/testing/selftests/bpf/progs/verifier_precision.c
index 6f325876efdd..f4459561bf39 100644
--- a/tools/testing/selftests/bpf/progs/verifier_precision.c
+++ b/tools/testing/selftests/bpf/progs/verifier_precision.c
@@ -2,8 +2,10 @@
/* Copyright (C) 2023 SUSE LLC */
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
+#include <stdbool.h>
#include "../../../include/linux/filter.h"
#include "bpf_misc.h"
+#include "bpf_kfuncs.h"
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
@@ -642,4 +644,102 @@ __naked int bpf_atomic_cmpxchg_32bit_precision(void)
: __clobber_all);
}
+/*
+ * Verification takes two paths: with r1 being scalar zero on path (1)
+ * and with r1 being some other scalar on path (2).
+ * Check that the verifier does not use checkpoints created
+ * on path (1) to prune path (2).
+ */
+SEC("?tc")
+__flag(BPF_F_TEST_STATE_FREQ)
+__failure __msg("R1 type=scalar expected=fp")
+__naked int null_mem_arg_zero_size(void)
+{
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "r1 = 42;"
+ "if r0 > 42 goto 1f;"
+ "r1 = 0;"
+ "1:"
+ "r2 = 0;"
+ "r3 = 0;"
+ "r4 = 0;"
+ "r5 = 0;"
+ /*
+ * ARG_PTR_TO_MEM | PTR_MAYBE_NULL parameter can be NULL,
+ * but can't be some other scalar value.
+ */
+ "call %[bpf_csum_diff];"
+ "r0 = 0;"
+ "exit;"
+ :
+ : __imm(bpf_get_prandom_u32),
+ __imm(bpf_csum_diff)
+ : __clobber_all);
+}
+
+__weak int subprog_mem_arg(int *p)
+{
+ if (p)
+ return *p;
+ return 0;
+}
+
+/*
+ * Verification takes two paths: with r1 being scalar zero on path (1)
+ * and with r1 being some other scalar on path (2).
+ * Check that the verifier does not use checkpoints created
+ * on path (1) to prune path (2).
+ */
+SEC("?raw_tp")
+__flag(BPF_F_TEST_STATE_FREQ)
+__failure __msg("R1 type=scalar expected=fp")
+__naked int null_mem_arg_global_subprog(void)
+{
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "r1 = 42;"
+ "if r0 > 42 goto 1f;"
+ "r1 = 0;"
+ "1:"
+ "call subprog_mem_arg;"
+ "r0 = 0;"
+ "exit;"
+ :
+ : __imm(bpf_get_prandom_u32)
+ : __clobber_all);
+}
+
+/* Same as above, check that path with r3 == 0 does not prune the path with r3 != 0 */
+SEC("?tc")
+__flag(BPF_F_TEST_STATE_FREQ)
+__failure __msg("R3 type=scalar expected=fp")
+int null_kfunc_arg_dynptr_slice(struct __sk_buff *skb)
+{
+ struct bpf_dynptr ptr;
+
+ bpf_dynptr_from_skb(skb, 0, &ptr);
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "r3 = 42;"
+ "if r0 > 42 goto 1f;"
+ "r3 = 0;"
+ "1:"
+ "r1 = %[ptr];"
+ "r2 = 0;"
+ "r4 = 8;"
+ "call %[bpf_dynptr_slice];"
+ :
+ : __imm_ptr(ptr),
+ __imm(bpf_get_prandom_u32),
+ __imm(bpf_dynptr_slice)
+ : __clobber_common);
+ return 0;
+}
+
+void __kfunc_btf_root(void)
+{
+ bpf_dynptr_slice(0, 0, 0, 0);
+}
+
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
index 663d15fc5fd2..256547048cc4 100644
--- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
+++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
@@ -380,13 +380,14 @@ SEC("socket")
__success __log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
/*
- * check that r0 and r5 have different IDs after 'if',
- * collect_linked_regs() can't tie more than 5 registers for a single insn.
+ * check that r5 is unlinked after 'if', collect_linked_regs() can't tie
+ * more than 5 registers for a single insn and the register compared by
+ * the jump is not exempt from that.
*/
-__msg("7: (25) if r0 > 0x7 goto pc+0 ; R0=scalar(id=1")
+__msg("7: (25) if r5 > 0x7 goto pc+0 ; R5=scalar(smin=")
__msg("12: (bf) r5 = r5 ; R5=scalar(id=2")
/* check that r{0-4} are marked precise after 'if' */
-__msg("frame0: regs=r0 stack= before 7: (25) if r0 > 0x7 goto pc+0")
+__msg("frame0: regs=r0 stack= before 7: (25) if r5 > 0x7 goto pc+0")
__msg("frame0: parent state regs=r0,r1,r2,r3,r4 stack=:")
__naked void linked_regs_too_many_regs(void)
{
@@ -400,8 +401,8 @@ __naked void linked_regs_too_many_regs(void)
"r3 = r0;"
"r4 = r0;"
"r5 = r0;"
- /* propagate range for r{0-5} */
- "if r0 > 7 goto +0;"
+ /* r{0-4} fill the record, r5 does not fit and is unlinked */
+ "if r5 > 7 goto +0;"
/* keep r{1-4} live */
"r1 = r1;"
"r2 = r2;"
diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
index 8b166c42c4e0..39a1766dae3f 100644
--- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
+++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
@@ -1403,6 +1403,46 @@ __naked void partial_fill_from_cleaned_pointer_spill(void)
::: __clobber_all);
}
+SEC("raw_tp")
+__failure
+__msg("access may be outside object bounds")
+__flag(BPF_F_TEST_STATE_FREQ)
+__naked void imprecise_scalar_spill_half_dead(void)
+{
+ asm volatile (
+ /*
+ * Fork two paths: the one explored first spills an imprecise zero,
+ * the one explored second, an imprecise non-zero scalar.
+ */
+ "call %[bpf_get_prandom_u32];"
+ "if r0 > 42 goto 1f;"
+ "r6 = 0;"
+ "goto 2f;"
+"1:"
+ /* causes out of bounds access on a second path. */
+ "r6 = 100500;"
+"2:"
+ /* Force a checkpoint before the spill. */
+ "goto +0;"
+ "*(u64 *)(r10 - 8) = r6;"
+ /*
+ * Force stack cleanup, only the low half of the spill is alive,
+ * so the dead high half is degraded to raw stack bytes.
+ * Buggy verifier converted it to STACK_ZERO w/o proper precision propagation.
+ */
+ "goto +0;"
+ "r7 = *(u32 *)(r10 - 4);"
+ /* Use r7 as an offset into a one-byte buffer. */
+ "r1 = %[single_byte_buf] ll;"
+ "r1 += r7;"
+ "r0 = *(u8 *)(r1 + 0);"
+ "exit;"
+:
+: __imm(bpf_get_prandom_u32),
+ __imm_addr(single_byte_buf)
+: __clobber_all);
+}
+
/* check valid spill/fill, ptr to tp buffer */
SEC("raw_tracepoint.w")
__success
diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c
index d21d32f6a676..dc0c7034c04f 100644
--- a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c
+++ b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c
@@ -287,9 +287,9 @@ __msg("17: (b7) r0 = 0")
__msg("18: (95) exit")
__msg("returning from callee:")
__msg("to caller at 9:")
-__msg("frame 0: propagating r1,r4")
+__msg("frame 0: propagating r1,r3,r4")
__msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1")
-__msg("mark_precise: frame0: regs=r1,r4 stack= before 18: (95) exit")
+__msg("mark_precise: frame0: regs=r1,r3,r4 stack= before 18: (95) exit")
__msg("from 18 to 9: safe")
__naked int callback_result_precise(void)
{
@@ -419,9 +419,9 @@ __msg("to caller at 9:")
/* r1, r4 are always precise for bpf_loop(),
* r6 was marked before backtracking to callback body.
*/
-__msg("frame 0: propagating r1,r4,r6")
+__msg("frame 0: propagating r1,r3,r4,r6")
__msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1")
-__msg("mark_precise: frame0: regs=r1,r4,r6 stack= before 16: (95) exit")
+__msg("mark_precise: frame0: regs=r1,r3,r4,r6 stack= before 16: (95) exit")
__msg("mark_precise: frame1: regs= stack= before 15: (b7) r0 = 0")
__msg("mark_precise: frame1: regs= stack= before 9: (85) call bpf_loop")
__msg("mark_precise: frame0: parent state regs= stack=:")
@@ -575,9 +575,9 @@ __msg("to caller at 10:")
/* r1, r4 are always precise for bpf_loop(),
* fp-8 was marked before backtracking to callback body.
*/
-__msg("frame 0: propagating r1,r4,fp-8")
+__msg("frame 0: propagating r1,r3,r4,fp-8")
__msg("mark_precise: frame0: last_idx 10 first_idx 10 subseq_idx -1")
-__msg("mark_precise: frame0: regs=r1,r4 stack=-8 before 18: (95) exit")
+__msg("mark_precise: frame0: regs=r1,r3,r4 stack=-8 before 18: (95) exit")
__msg("mark_precise: frame1: regs= stack= before 17: (b7) r0 = 0")
__msg("mark_precise: frame1: regs= stack= before 10: (85) call bpf_loop#181")
__msg("mark_precise: frame0: parent state regs= stack=:")
@@ -846,4 +846,55 @@ __naked int subprog_result_tail_call(void)
);
}
+__naked __noinline __used
+static int ld_abs_subprog(void)
+{
+ asm volatile (
+ "r6 = r1;"
+ "r7 = r1;"
+ ".8byte %[ld_abs];"
+ "exit;"
+ :
+ : __imm_insn(ld_abs, BPF_LD_ABS(BPF_W, 0))
+ : __clobber_all);
+}
+
+/*
+ * Buggy verifier did not properly backtrack early subprogram exit
+ * modelled for BPF_LD | BPF_ABS instruction, causing a segfault.
+ */
+SEC("socket")
+__success
+__log_level(2)
+/* early exit path */
+__msg("3: (0f) r1 += r7")
+__msg("mark_precise: frame0: regs=r7 stack= before 2: (bf) r1 = r10")
+__msg("mark_precise: frame0: regs=r7 stack= before 9: (20) r0 = *(u32 *)skb[0]")
+__msg("mark_precise: frame1: regs= stack= before 8: (bf) r7 = r1")
+__msg("mark_precise: frame1: regs= stack= before 7: (bf) r6 = r1")
+__msg("mark_precise: frame1: regs= stack= before 1: (85) call pc+5")
+__msg("mark_precise: frame0: regs=r7 stack= before 0: (b7) r7 = -8")
+/* fallthrough path */
+__msg("3: (0f) r1 += r7")
+__msg("mark_precise: frame0: regs=r7 stack= before 2: (bf) r1 = r10")
+__msg("mark_precise: frame0: regs=r7 stack= before 10: (95) exit")
+__msg("mark_precise: frame1: regs= stack= before 9: (20) r0 = *(u32 *)skb[0]")
+__msg("mark_precise: frame1: regs= stack= before 8: (bf) r7 = r1")
+__msg("mark_precise: frame1: regs= stack= before 7: (bf) r6 = r1")
+__msg("mark_precise: frame1: regs= stack= before 1: (85) call pc+5")
+__msg("mark_precise: frame0: regs=r7 stack= before 0: (b7) r7 = -8")
+__naked int ld_abs_backtrack_both_paths(void)
+{
+ asm volatile (
+ "r7 = -8;"
+ "call ld_abs_subprog;"
+ "r1 = r10;"
+ "r1 += r7;" /* mark r7 as precise */
+ "*(u64 *)(r1 + 0) = 0;"
+ "r0 = 0;"
+ "exit;"
+ ::: __clobber_all
+ );
+}
+
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_zext.c b/tools/testing/selftests/bpf/progs/verifier_zext.c
index 8f2362da91d6..572017fe28fb 100644
--- a/tools/testing/selftests/bpf/progs/verifier_zext.c
+++ b/tools/testing/selftests/bpf/progs/verifier_zext.c
@@ -356,6 +356,32 @@ __naked void arena_ptr(void)
: __clobber_all);
}
+/*
+ * Result of a 32-bit cmpxchg is always explicitly zero extended.
+ * Check that this holds for arenas (BPF_PROBE_ATOMIC instruction flavor).
+ */
+SEC("socket")
+__success
+__xlated("probe r0 = atomic_cmpxchg((u32 *)(r1 +0), r0, r2)")
+__xlated("w0 = w0")
+__naked void zext_arena_cmpxchg32(void)
+{
+ asm volatile (" \
+ r9 = %[arena] ll; /* associate the arena with the program */ \
+ r1 = 0; \
+ r1 = addr_space_cast(r1, 0, 1); \
+ r0 = 0; \
+ r2 = 0; \
+ .8byte %[cmpxchg32]; \
+ r0 >>= 32; /* make the upper half live */ \
+ exit; \
+" :
+ : __imm_addr(arena),
+ __imm_insn(cmpxchg32,
+ BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_1, BPF_REG_2, 0))
+ : __clobber_all);
+}
+
#endif
/* Check if probe mem loads keep their zero extension. */
diff --git a/tools/testing/selftests/bpf/verifier/pseudo_func.c b/tools/testing/selftests/bpf/verifier/pseudo_func.c
new file mode 100644
index 000000000000..63c5c67d51de
--- /dev/null
+++ b/tools/testing/selftests/bpf/verifier/pseudo_func.c
@@ -0,0 +1,45 @@
+/*
+ * Buggy verifier accepted the program below while not patching BPF_PSEUDO_FUNC
+ * load instruction to contain a real address. Which resulted in a function call
+ * to a bogus address.
+ */
+{
+ "BPF_PSEUDO_FUNC reference to the main program",
+ .insns = {
+ /* r6 = bpf_map_lookup_elem(&timer_map, &(int){0}); */
+ BPF_ST_MEM(BPF_W, BPF_REG_10, -4, 0),
+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4),
+ BPF_LD_MAP_FD(BPF_REG_1, 0),
+ BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 10),
+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_0),
+ /* bpf_timer_init(r6, &timer_map, 0); */
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
+ BPF_LD_MAP_FD(BPF_REG_2, 0),
+ BPF_MOV64_IMM(BPF_REG_3, 0),
+ BPF_EMIT_CALL(BPF_FUNC_timer_init),
+ /* bpf_timer_set_callback(r6, <insn #0>); */
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
+ BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, BPF_REG_2, BPF_PSEUDO_FUNC, 0, -15),
+ BPF_RAW_INSN(0, 0, 0, 0, 0),
+ BPF_EMIT_CALL(BPF_FUNC_timer_set_callback),
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_EXIT_INSN(),
+ },
+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
+ .fixup_map_timer = { 3, 9 },
+ .result = REJECT,
+ .errstr = "callback function cannot be the main program",
+ .func_info = { { 0, 4 /* main_prog */ } },
+ .func_info_cnt = 1,
+ .btf_strings = "\0int\0ctx\0main_prog",
+ .btf_types = {
+ /* 1: int */ BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
+ /* 2: void* */ BTF_PTR_ENC(0),
+ /* 3: int __(void *) */ BTF_FUNC_PROTO_ENC(1, 1),
+ BTF_FUNC_PROTO_ARG_ENC(5, 2),
+ /* 4: main_prog */ BTF_FUNC_ENC(9, 3),
+ BTF_END_RAW
+ }
+},