diff mbox series

[bpf-next,v8,8/9] bpf: Support private stack for struct_ops progs

Message ID 20241101031032.2680930-1-yonghong.song@linux.dev (mailing list archive)
State Superseded
Delegated to: BPF
Headers show
Series bpf: Support private stack for bpf progs | expand

Checks

Context Check Description
bpf/vmtest-bpf-next-PR success PR summary
bpf/vmtest-bpf-next-VM_Test-30 success Logs for x86_64-llvm-17 / test (test_progs, false, 360) / test_progs on x86_64 with llvm-17
netdev/series_format success Posting correctly formatted
netdev/tree_selection success Clearly marked for bpf-next, async
netdev/ynl success Generated files up to date; no warnings/errors; no diff in generated;
netdev/fixes_present success Fixes tag not required for -next series
netdev/header_inline success No static functions without inline keyword in header files
netdev/build_32bit success Errors and warnings before: 204 this patch: 204
netdev/build_tools success Errors and warnings before: 0 (+0) this patch: 0 (+0)
netdev/cc_maintainers warning 8 maintainers not CCed: sdf@fomichev.me kpsingh@kernel.org john.fastabend@gmail.com eddyz87@gmail.com martin.lau@linux.dev song@kernel.org haoluo@google.com jolsa@kernel.org
netdev/build_clang success Errors and warnings before: 253 this patch: 253
netdev/verify_signedoff success Signed-off-by tag matches author and committer
netdev/deprecated_api success None detected
netdev/check_selftest success No net selftest shell script
netdev/verify_fixes success No Fixes tag
netdev/build_allmodconfig_warn success Errors and warnings before: 6962 this patch: 6962
netdev/checkpatch warning WARNING: line length of 96 exceeds 80 columns
netdev/build_clang_rust success No Rust files in patch. Skipping build
netdev/kdoc success Errors and warnings before: 6 this patch: 6
netdev/source_inline success Was 0 now: 0
bpf/vmtest-bpf-next-VM_Test-12 success Logs for s390x-gcc / build-release
bpf/vmtest-bpf-next-VM_Test-4 success Logs for aarch64-gcc / build / build for aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-9 success Logs for aarch64-gcc / test (test_verifier, false, 360) / test_verifier on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-11 success Logs for s390x-gcc / build / build for s390x with gcc
bpf/vmtest-bpf-next-VM_Test-6 success Logs for aarch64-gcc / test (test_maps, false, 360) / test_maps on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-16 success Logs for s390x-gcc / veristat
bpf/vmtest-bpf-next-VM_Test-17 success Logs for set-matrix
bpf/vmtest-bpf-next-VM_Test-18 success Logs for x86_64-gcc / build / build for x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-19 success Logs for x86_64-gcc / build-release
bpf/vmtest-bpf-next-VM_Test-27 success Logs for x86_64-llvm-17 / build / build for x86_64 with llvm-17
bpf/vmtest-bpf-next-VM_Test-28 success Logs for x86_64-llvm-17 / build-release / build for x86_64 with llvm-17-O2
bpf/vmtest-bpf-next-VM_Test-33 success Logs for x86_64-llvm-17 / veristat
bpf/vmtest-bpf-next-VM_Test-34 success Logs for x86_64-llvm-18 / build / build for x86_64 with llvm-18
bpf/vmtest-bpf-next-VM_Test-35 success Logs for x86_64-llvm-18 / build-release / build for x86_64 with llvm-18-O2
bpf/vmtest-bpf-next-VM_Test-41 success Logs for x86_64-llvm-18 / veristat
bpf/vmtest-bpf-next-VM_Test-13 success Logs for s390x-gcc / test (test_progs, false, 360) / test_progs on s390x with gcc
bpf/vmtest-bpf-next-VM_Test-15 success Logs for s390x-gcc / test (test_verifier, false, 360) / test_verifier on s390x with gcc
bpf/vmtest-bpf-next-VM_Test-14 success Logs for s390x-gcc / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on s390x with gcc
bpf/vmtest-bpf-next-VM_Test-20 success Logs for x86_64-gcc / test (test_maps, false, 360) / test_maps on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-25 success Logs for x86_64-gcc / test (test_verifier, false, 360) / test_verifier on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-29 success Logs for x86_64-llvm-17 / test (test_maps, false, 360) / test_maps on x86_64 with llvm-17
bpf/vmtest-bpf-next-VM_Test-32 success Logs for x86_64-llvm-17 / test (test_verifier, false, 360) / test_verifier on x86_64 with llvm-17
bpf/vmtest-bpf-next-VM_Test-40 success Logs for x86_64-llvm-18 / test (test_verifier, false, 360) / test_verifier on x86_64 with llvm-18
bpf/vmtest-bpf-next-VM_Test-21 success Logs for x86_64-gcc / test (test_progs, false, 360) / test_progs on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-22 success Logs for x86_64-gcc / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-23 success Logs for x86_64-gcc / test (test_progs_no_alu32_parallel, true, 30) / test_progs_no_alu32_parallel on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-24 success Logs for x86_64-gcc / test (test_progs_parallel, true, 30) / test_progs_parallel on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-26 success Logs for x86_64-gcc / veristat / veristat on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-31 success Logs for x86_64-llvm-17 / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on x86_64 with llvm-17
bpf/vmtest-bpf-next-VM_Test-36 success Logs for x86_64-llvm-18 / test (test_maps, false, 360) / test_maps on x86_64 with llvm-18
bpf/vmtest-bpf-next-VM_Test-37 success Logs for x86_64-llvm-18 / test (test_progs, false, 360) / test_progs on x86_64 with llvm-18
bpf/vmtest-bpf-next-VM_Test-38 success Logs for x86_64-llvm-18 / test (test_progs_cpuv4, false, 360) / test_progs_cpuv4 on x86_64 with llvm-18
bpf/vmtest-bpf-next-VM_Test-39 success Logs for x86_64-llvm-18 / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on x86_64 with llvm-18
bpf/vmtest-bpf-next-VM_Test-2 success Logs for Unittests
bpf/vmtest-bpf-next-VM_Test-3 success Logs for Validate matrix.py
bpf/vmtest-bpf-next-VM_Test-1 success Logs for ShellCheck
bpf/vmtest-bpf-next-VM_Test-0 success Logs for Lint
bpf/vmtest-bpf-next-VM_Test-10 success Logs for x86_64-gcc / build-release
bpf/vmtest-bpf-next-VM_Test-8 success Logs for set-matrix
bpf/vmtest-bpf-next-VM_Test-7 success Logs for s390x-gcc / build-release
bpf/vmtest-bpf-next-VM_Test-5 success Logs for aarch64-gcc / build-release

Commit Message

Yonghong Song Nov. 1, 2024, 3:10 a.m. UTC 
For struct_ops progs, whether a particular prog will use private stack
or not (prog->aux->use_priv_stack) will be set before actual insn-level
verification for that prog. One particular implementation is to
piggyback on struct_ops->check_member(). The next patch will have an
example for this. The struct_ops->check_member() will set
prog->aux->use_priv_stack to be true which enables private stack
usage with ignoring BPF_PRIV_STACK_MIN_SIZE limit.

If use_priv_stack is true for a particular struct_ops prog, bpf
trampoline will need to do recursion checks (one level at this point)
to avoid stack overwrite. A field (recursion_skipped()) is added to
bpf_prog_aux structure such that if bpf_prog->aux->recursion_skipped
is set by the struct_ops subsystem, the function will be called
to terminate the prog run, collect related info, etc.

Acked-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
 include/linux/bpf.h          |  1 +
 include/linux/bpf_verifier.h |  1 +
 kernel/bpf/trampoline.c      |  4 ++++
 kernel/bpf/verifier.c        | 36 ++++++++++++++++++++++++++++++++----
 4 files changed, 38 insertions(+), 4 deletions(-)

Comments

Alexei Starovoitov Nov. 1, 2024, 8:13 p.m. UTC | #1 
On Thu, Oct 31, 2024 at 8:10 PM Yonghong Song <yonghong.song@linux.dev> wrote:
>
> For struct_ops progs, whether a particular prog will use private stack
> or not (prog->aux->use_priv_stack) will be set before actual insn-level
> verification for that prog. One particular implementation is to
> piggyback on struct_ops->check_member(). The next patch will have an
> example for this. The struct_ops->check_member() will set
> prog->aux->use_priv_stack to be true which enables private stack
> usage with ignoring BPF_PRIV_STACK_MIN_SIZE limit.
>
> If use_priv_stack is true for a particular struct_ops prog, bpf
> trampoline will need to do recursion checks (one level at this point)
> to avoid stack overwrite. A field (recursion_skipped()) is added to
> bpf_prog_aux structure such that if bpf_prog->aux->recursion_skipped
> is set by the struct_ops subsystem, the function will be called
> to terminate the prog run, collect related info, etc.
>
> Acked-by: Tejun Heo <tj@kernel.org>
> Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
> ---
>  include/linux/bpf.h          |  1 +
>  include/linux/bpf_verifier.h |  1 +
>  kernel/bpf/trampoline.c      |  4 ++++
>  kernel/bpf/verifier.c        | 36 ++++++++++++++++++++++++++++++++----
>  4 files changed, 38 insertions(+), 4 deletions(-)
>
> diff --git a/include/linux/bpf.h b/include/linux/bpf.h
> index 8a3ea7440a4a..7a34108c6974 100644
> --- a/include/linux/bpf.h
> +++ b/include/linux/bpf.h
> @@ -1528,6 +1528,7 @@ struct bpf_prog_aux {
>         u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
>         struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
>         struct bpf_arena *arena;
> +       void (*recursion_skipped)(struct bpf_prog *prog); /* callback if recursion is skipped */

The name doesn't fit.
The recursion wasn't skipped.
It's the execution of the program that was skipped.
'recursion_detected' or 'recursion_disallowed' would be a better name.

>         /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
>         const struct btf_type *attach_func_proto;
>         /* function name for valid attach_btf_id */
> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> index bc28ce7996ac..ff0fba935f89 100644
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h
> @@ -889,6 +889,7 @@ static inline bool bpf_prog_check_recur(const struct bpf_prog *prog)
>         case BPF_PROG_TYPE_TRACING:
>                 return prog->expected_attach_type != BPF_TRACE_ITER;
>         case BPF_PROG_TYPE_STRUCT_OPS:
> +               return prog->aux->use_priv_stack;
>         case BPF_PROG_TYPE_LSM:
>                 return false;
>         default:
> diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
> index 9f36c049f4c2..a84e60efbf89 100644
> --- a/kernel/bpf/trampoline.c
> +++ b/kernel/bpf/trampoline.c
> @@ -899,6 +899,8 @@ static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tram
>
>         if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
>                 bpf_prog_inc_misses_counter(prog);
> +               if (prog->aux->recursion_skipped)
> +                       prog->aux->recursion_skipped(prog);
>                 return 0;
>         }
>         return bpf_prog_start_time();
> @@ -975,6 +977,8 @@ u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
>
>         if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
>                 bpf_prog_inc_misses_counter(prog);
> +               if (prog->aux->recursion_skipped)
> +                       prog->aux->recursion_skipped(prog);
>                 return 0;
>         }
>         return bpf_prog_start_time();
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index 30e74db6a85f..865191c5d21b 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -6023,17 +6023,31 @@ static int check_ptr_alignment(struct bpf_verifier_env *env,
>
>  static int bpf_enable_priv_stack(struct bpf_verifier_env *env)
>  {
> +       bool force_priv_stack = env->prog->aux->use_priv_stack;
>         struct bpf_subprog_info *si;
> +       int ret;
> +
> +       if (!bpf_jit_supports_private_stack()) {
> +               if (force_priv_stack) {
> +                       verbose(env, "Private stack not supported by jit\n");
> +                       return -EACCES;
> +               }

This logic would fit better in the patch 2.
Less code churn and the whole approach is easier to understand.

I don't like this inband signaling.
Now I see why you had that weird <0 check in patch 2 :(
This is ugly.
May be it should be a separate bool request_priv_stack:1
that struct_ops callback will set and it will clean up
this logic.

>
> -       if (!bpf_jit_supports_private_stack())
>                 return NO_PRIV_STACK;
> +       }
>
> +       ret = PRIV_STACK_ADAPTIVE;
>         switch (env->prog->type) {
>         case BPF_PROG_TYPE_KPROBE:
>         case BPF_PROG_TYPE_TRACEPOINT:
>         case BPF_PROG_TYPE_PERF_EVENT:
>         case BPF_PROG_TYPE_RAW_TRACEPOINT:
>                 break;
> +       case BPF_PROG_TYPE_STRUCT_OPS:
> +               if (!force_priv_stack)
> +                       return NO_PRIV_STACK;
> +               ret = PRIV_STACK_ALWAYS;
> +               break;
>         case BPF_PROG_TYPE_TRACING:
>                 if (env->prog->expected_attach_type != BPF_TRACE_ITER)
>                         break;
> @@ -6044,11 +6058,18 @@ static int bpf_enable_priv_stack(struct bpf_verifier_env *env)
>
>         si = env->subprog_info;
>         for (int i = 0; i < env->subprog_cnt; i++) {
> -               if (si[i].has_tail_call)
> +               if (si[i].has_tail_call) {
> +                       if (ret == PRIV_STACK_ALWAYS) {
> +                               verbose(env,
> +                                       "Private stack not supported due to tail call presence\n");
> +                               return -EACCES;

> +                       }
> +
>                         return NO_PRIV_STACK;
> +               }
>         }
>
> -       return PRIV_STACK_ADAPTIVE;
> +       return ret;
>  }
>
>  static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)
> @@ -6121,7 +6142,8 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
>                                         idx, subprog_depth);
>                                 return -EACCES;
>                         }
> -                       if (subprog_depth >= BPF_PRIV_STACK_MIN_SIZE) {
> +                       if (priv_stack_supported == PRIV_STACK_ALWAYS ||
> +                           subprog_depth >= BPF_PRIV_STACK_MIN_SIZE) {
>                                 subprog[idx].use_priv_stack = true;
>                                 subprog_visited[idx] = 1;
>                         }
> @@ -6271,6 +6293,12 @@ static int check_max_stack_depth(struct bpf_verifier_env *env)
>                                 depth_frame, subtree_depth);
>                         return -EACCES;
>                 }
> +               if (orig_priv_stack_supported == PRIV_STACK_ALWAYS) {
> +                       verbose(env,
> +                               "Private stack not supported due to possible nested subprog run\n");
> +                       ret = -EACCES;
> +                       goto out;
> +               }
>                 if (orig_priv_stack_supported == PRIV_STACK_ADAPTIVE) {
>                         for (int i = 0; i < env->subprog_cnt; i++)
>                                 si[i].use_priv_stack = false;
> --
> 2.43.5
>
Yonghong Song Nov. 4, 2024, 12:35 a.m. UTC | #2 
On 11/1/24 1:13 PM, Alexei Starovoitov wrote:
> On Thu, Oct 31, 2024 at 8:10 PM Yonghong Song <yonghong.song@linux.dev> wrote:
>> For struct_ops progs, whether a particular prog will use private stack
>> or not (prog->aux->use_priv_stack) will be set before actual insn-level
>> verification for that prog. One particular implementation is to
>> piggyback on struct_ops->check_member(). The next patch will have an
>> example for this. The struct_ops->check_member() will set
>> prog->aux->use_priv_stack to be true which enables private stack
>> usage with ignoring BPF_PRIV_STACK_MIN_SIZE limit.
>>
>> If use_priv_stack is true for a particular struct_ops prog, bpf
>> trampoline will need to do recursion checks (one level at this point)
>> to avoid stack overwrite. A field (recursion_skipped()) is added to
>> bpf_prog_aux structure such that if bpf_prog->aux->recursion_skipped
>> is set by the struct_ops subsystem, the function will be called
>> to terminate the prog run, collect related info, etc.
>>
>> Acked-by: Tejun Heo <tj@kernel.org>
>> Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
>> ---
>>   include/linux/bpf.h          |  1 +
>>   include/linux/bpf_verifier.h |  1 +
>>   kernel/bpf/trampoline.c      |  4 ++++
>>   kernel/bpf/verifier.c        | 36 ++++++++++++++++++++++++++++++++----
>>   4 files changed, 38 insertions(+), 4 deletions(-)
>>
>> diff --git a/include/linux/bpf.h b/include/linux/bpf.h
>> index 8a3ea7440a4a..7a34108c6974 100644
>> --- a/include/linux/bpf.h
>> +++ b/include/linux/bpf.h
>> @@ -1528,6 +1528,7 @@ struct bpf_prog_aux {
>>          u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
>>          struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
>>          struct bpf_arena *arena;
>> +       void (*recursion_skipped)(struct bpf_prog *prog); /* callback if recursion is skipped */
> The name doesn't fit.
> The recursion wasn't skipped.
> It's the execution of the program that was skipped.
> 'recursion_detected' or 'recursion_disallowed' would be a better name.

I will use recursion_detected().

>
>>          /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
>>          const struct btf_type *attach_func_proto;
>>          /* function name for valid attach_btf_id */
>> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
>> index bc28ce7996ac..ff0fba935f89 100644
>> --- a/include/linux/bpf_verifier.h
>> +++ b/include/linux/bpf_verifier.h
>> @@ -889,6 +889,7 @@ static inline bool bpf_prog_check_recur(const struct bpf_prog *prog)
>>          case BPF_PROG_TYPE_TRACING:
>>                  return prog->expected_attach_type != BPF_TRACE_ITER;
>>          case BPF_PROG_TYPE_STRUCT_OPS:
>> +               return prog->aux->use_priv_stack;
>>          case BPF_PROG_TYPE_LSM:
>>                  return false;
>>          default:
>> diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
>> index 9f36c049f4c2..a84e60efbf89 100644
>> --- a/kernel/bpf/trampoline.c
>> +++ b/kernel/bpf/trampoline.c
>> @@ -899,6 +899,8 @@ static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tram
>>
>>          if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
>>                  bpf_prog_inc_misses_counter(prog);
>> +               if (prog->aux->recursion_skipped)
>> +                       prog->aux->recursion_skipped(prog);
>>                  return 0;
>>          }
>>          return bpf_prog_start_time();
>> @@ -975,6 +977,8 @@ u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
>>
>>          if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
>>                  bpf_prog_inc_misses_counter(prog);
>> +               if (prog->aux->recursion_skipped)
>> +                       prog->aux->recursion_skipped(prog);
>>                  return 0;
>>          }
>>          return bpf_prog_start_time();
>> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
>> index 30e74db6a85f..865191c5d21b 100644
>> --- a/kernel/bpf/verifier.c
>> +++ b/kernel/bpf/verifier.c
>> @@ -6023,17 +6023,31 @@ static int check_ptr_alignment(struct bpf_verifier_env *env,
>>
>>   static int bpf_enable_priv_stack(struct bpf_verifier_env *env)
>>   {
>> +       bool force_priv_stack = env->prog->aux->use_priv_stack;
>>          struct bpf_subprog_info *si;
>> +       int ret;
>> +
>> +       if (!bpf_jit_supports_private_stack()) {
>> +               if (force_priv_stack) {
>> +                       verbose(env, "Private stack not supported by jit\n");
>> +                       return -EACCES;
>> +               }
> This logic would fit better in the patch 2.
> Less code churn and the whole approach is easier to understand.
>
> I don't like this inband signaling.
> Now I see why you had that weird <0 check in patch 2 :(
> This is ugly.
> May be it should be a separate bool request_priv_stack:1
> that struct_ops callback will set and it will clean up
> this logic.

I can add this logic to function check_struct_ops_btf_id(),
which is struct_ops preparation for verification. This
will ensure in bpf_enable_priv_stack(), if
!bpf_jit_supports_private_stack(), it is guaranteed to
return NO_PRIV_STACK.

>
>> -       if (!bpf_jit_supports_private_stack())
>>                  return NO_PRIV_STACK;
>> +       }
>>
>> +       ret = PRIV_STACK_ADAPTIVE;
>>          switch (env->prog->type) {
>>          case BPF_PROG_TYPE_KPROBE:
>>          case BPF_PROG_TYPE_TRACEPOINT:
>>          case BPF_PROG_TYPE_PERF_EVENT:
>>          case BPF_PROG_TYPE_RAW_TRACEPOINT:
>>                  break;
>> +       case BPF_PROG_TYPE_STRUCT_OPS:
>> +               if (!force_priv_stack)
>> +                       return NO_PRIV_STACK;
>> +               ret = PRIV_STACK_ALWAYS;
>> +               break;
>>          case BPF_PROG_TYPE_TRACING:
>>                  if (env->prog->expected_attach_type != BPF_TRACE_ITER)
>>                          break;
>> @@ -6044,11 +6058,18 @@ static int bpf_enable_priv_stack(struct bpf_verifier_env *env)
>>
>>          si = env->subprog_info;
>>          for (int i = 0; i < env->subprog_cnt; i++) {
>> -               if (si[i].has_tail_call)
>> +               if (si[i].has_tail_call) {
>> +                       if (ret == PRIV_STACK_ALWAYS) {
>> +                               verbose(env,
>> +                                       "Private stack not supported due to tail call presence\n");
>> +                               return -EACCES;
>> +                       }
>> +
>>                          return NO_PRIV_STACK;
>> +               }
>>          }
>>
>> -       return PRIV_STACK_ADAPTIVE;
>> +       return ret;
>>   }
>>
[...]
diff mbox series

Patch

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 8a3ea7440a4a..7a34108c6974 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1528,6 +1528,7 @@  struct bpf_prog_aux {
 	u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
 	struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
 	struct bpf_arena *arena;
+	void (*recursion_skipped)(struct bpf_prog *prog); /* callback if recursion is skipped */
 	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
 	const struct btf_type *attach_func_proto;
 	/* function name for valid attach_btf_id */
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index bc28ce7996ac..ff0fba935f89 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -889,6 +889,7 @@  static inline bool bpf_prog_check_recur(const struct bpf_prog *prog)
 	case BPF_PROG_TYPE_TRACING:
 		return prog->expected_attach_type != BPF_TRACE_ITER;
 	case BPF_PROG_TYPE_STRUCT_OPS:
+		return prog->aux->use_priv_stack;
 	case BPF_PROG_TYPE_LSM:
 		return false;
 	default:
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 9f36c049f4c2..a84e60efbf89 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -899,6 +899,8 @@  static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tram
 
 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
 		bpf_prog_inc_misses_counter(prog);
+		if (prog->aux->recursion_skipped)
+			prog->aux->recursion_skipped(prog);
 		return 0;
 	}
 	return bpf_prog_start_time();
@@ -975,6 +977,8 @@  u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
 
 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
 		bpf_prog_inc_misses_counter(prog);
+		if (prog->aux->recursion_skipped)
+			prog->aux->recursion_skipped(prog);
 		return 0;
 	}
 	return bpf_prog_start_time();
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 30e74db6a85f..865191c5d21b 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -6023,17 +6023,31 @@  static int check_ptr_alignment(struct bpf_verifier_env *env,
 
 static int bpf_enable_priv_stack(struct bpf_verifier_env *env)
 {
+	bool force_priv_stack = env->prog->aux->use_priv_stack;
 	struct bpf_subprog_info *si;
+	int ret;
+
+	if (!bpf_jit_supports_private_stack()) {
+		if (force_priv_stack) {
+			verbose(env, "Private stack not supported by jit\n");
+			return -EACCES;
+		}
 
-	if (!bpf_jit_supports_private_stack())
 		return NO_PRIV_STACK;
+	}
 
+	ret = PRIV_STACK_ADAPTIVE;
 	switch (env->prog->type) {
 	case BPF_PROG_TYPE_KPROBE:
 	case BPF_PROG_TYPE_TRACEPOINT:
 	case BPF_PROG_TYPE_PERF_EVENT:
 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
 		break;
+	case BPF_PROG_TYPE_STRUCT_OPS:
+		if (!force_priv_stack)
+			return NO_PRIV_STACK;
+		ret = PRIV_STACK_ALWAYS;
+		break;
 	case BPF_PROG_TYPE_TRACING:
 		if (env->prog->expected_attach_type != BPF_TRACE_ITER)
 			break;
@@ -6044,11 +6058,18 @@  static int bpf_enable_priv_stack(struct bpf_verifier_env *env)
 
 	si = env->subprog_info;
 	for (int i = 0; i < env->subprog_cnt; i++) {
-		if (si[i].has_tail_call)
+		if (si[i].has_tail_call) {
+			if (ret == PRIV_STACK_ALWAYS) {
+				verbose(env,
+					"Private stack not supported due to tail call presence\n");
+				return -EACCES;
+			}
+
 			return NO_PRIV_STACK;
+		}
 	}
 
-	return PRIV_STACK_ADAPTIVE;
+	return ret;
 }
 
 static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)
@@ -6121,7 +6142,8 @@  static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 					idx, subprog_depth);
 				return -EACCES;
 			}
-			if (subprog_depth >= BPF_PRIV_STACK_MIN_SIZE) {
+			if (priv_stack_supported == PRIV_STACK_ALWAYS ||
+			    subprog_depth >= BPF_PRIV_STACK_MIN_SIZE) {
 				subprog[idx].use_priv_stack = true;
 				subprog_visited[idx] = 1;
 			}
@@ -6271,6 +6293,12 @@  static int check_max_stack_depth(struct bpf_verifier_env *env)
 				depth_frame, subtree_depth);
 			return -EACCES;
 		}
+		if (orig_priv_stack_supported == PRIV_STACK_ALWAYS) {
+			verbose(env,
+				"Private stack not supported due to possible nested subprog run\n");
+			ret = -EACCES;
+			goto out;
+		}
 		if (orig_priv_stack_supported == PRIV_STACK_ADAPTIVE) {
 			for (int i = 0; i < env->subprog_cnt; i++)
 				si[i].use_priv_stack = false;