diff mbox series

[bpf-next,5/5] bpf: trampoline: support FTRACE_OPS_FL_SHARE_IPMODIFY

Message ID 20220601175749.3071572-6-song@kernel.org (mailing list archive)
State Superseded
Delegated to: BPF
Headers show
Series ftrace: host klp and bpf trampoline together | expand

Checks

Context Check Description
netdev/tree_selection success Clearly marked for bpf-next, async
netdev/fixes_present success Fixes tag not required for -next series
netdev/subject_prefix success Link
netdev/cover_letter success Series has a cover letter
netdev/patch_count success Link
netdev/header_inline success No static functions without inline keyword in header files
netdev/build_32bit success Errors and warnings before: 1444 this patch: 1444
netdev/cc_maintainers warning 5 maintainers not CCed: songliubraving@fb.com yhs@fb.com john.fastabend@gmail.com kafai@fb.com kpsingh@kernel.org
netdev/build_clang success Errors and warnings before: 177 this patch: 177
netdev/module_param success Was 0 now: 0
netdev/verify_signedoff success Signed-off-by tag matches author and committer
netdev/check_selftest success No net selftest shell script
netdev/verify_fixes success No Fixes tag
netdev/build_allmodconfig_warn success Errors and warnings before: 1451 this patch: 1451
netdev/checkpatch warning CHECK: Please don't use multiple blank lines WARNING: line length of 83 exceeds 80 columns WARNING: line length of 85 exceeds 80 columns WARNING: line length of 90 exceeds 80 columns WARNING: line length of 92 exceeds 80 columns
netdev/kdoc success Errors and warnings before: 0 this patch: 0
netdev/source_inline success Was 0 now: 0
bpf/vmtest-bpf-next-VM_Test-1 success Logs for Kernel LATEST on ubuntu-latest with gcc
bpf/vmtest-bpf-next-VM_Test-2 success Logs for Kernel LATEST on ubuntu-latest with llvm-15
bpf/vmtest-bpf-next-PR fail PR summary
bpf/vmtest-bpf-next-VM_Test-3 fail Logs for Kernel LATEST on z15 with gcc

Commit Message

Song Liu June 1, 2022, 5:57 p.m. UTC 
This allows bpf trampoline to trace kernel functions with live patch.
Also, move bpf trampoline to *_ftrace_direct_multi APIs, which allows
setting different flags of ftrace_ops.

Signed-off-by: Song Liu <song@kernel.org>
---
 include/linux/bpf.h     |   3 ++
 kernel/bpf/trampoline.c | 100 +++++++++++++++++++++++++++++++++++-----
 2 files changed, 91 insertions(+), 12 deletions(-)

Comments

kernel test robot June 1, 2022, 10:44 p.m. UTC | #1 
Hi Song,

I love your patch! Perhaps something to improve:

[auto build test WARNING on bpf-next/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Song-Liu/ftrace-host-klp-and-bpf-trampoline-together/20220602-020112
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
config: i386-randconfig-a015 (https://download.01.org/0day-ci/archive/20220602/202206020622.HnFjEObo-lkp@intel.com/config)
compiler: clang version 15.0.0 (https://github.com/llvm/llvm-project c825abd6b0198fb088d9752f556a70705bc99dfd)
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/intel-lab-lkp/linux/commit/7edcf1c49617641579f2bc36b86c7d59bea20aef
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review Song-Liu/ftrace-host-klp-and-bpf-trampoline-together/20220602-020112
        git checkout 7edcf1c49617641579f2bc36b86c7d59bea20aef
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=i386 SHELL=/bin/bash kernel/bpf/

If you fix the issue, kindly add following tag where applicable
Reported-by: kernel test robot <lkp@intel.com>

All warnings (new ones prefixed by >>):

>> kernel/bpf/trampoline.c:30:66: warning: declaration of 'enum ftrace_ops_cmd' will not be visible outside of this function [-Wvisibility]
   static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *op, enum ftrace_ops_cmd cmd);
                                                                    ^
   kernel/bpf/trampoline.c:92:21: error: invalid application of 'sizeof' to an incomplete type 'struct ftrace_ops'
           tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);
                              ^     ~~~~~~~~~~~~~~~~~~~
   include/linux/bpf.h:47:8: note: forward declaration of 'struct ftrace_ops'
   struct ftrace_ops;
          ^
   kernel/bpf/trampoline.c:100:10: error: incomplete definition of type 'struct ftrace_ops'
           tr->fops->private = tr;
           ~~~~~~~~^
   include/linux/bpf.h:47:8: note: forward declaration of 'struct ftrace_ops'
   struct ftrace_ops;
          ^
   kernel/bpf/trampoline.c:101:10: error: incomplete definition of type 'struct ftrace_ops'
           tr->fops->ops_func = bpf_tramp_ftrace_ops_func;
           ~~~~~~~~^
   include/linux/bpf.h:47:8: note: forward declaration of 'struct ftrace_ops'
   struct ftrace_ops;
          ^
   kernel/bpf/trampoline.c:397:11: error: incomplete definition of type 'struct ftrace_ops'
                   tr->fops->flags |= FTRACE_OPS_FL_SHARE_IPMODIFY;
                   ~~~~~~~~^
   include/linux/bpf.h:47:8: note: forward declaration of 'struct ftrace_ops'
   struct ftrace_ops;
          ^
   kernel/bpf/trampoline.c:397:22: error: use of undeclared identifier 'FTRACE_OPS_FL_SHARE_IPMODIFY'
                   tr->fops->flags |= FTRACE_OPS_FL_SHARE_IPMODIFY;
                                      ^
   kernel/bpf/trampoline.c:415:11: error: incomplete definition of type 'struct ftrace_ops'
                   tr->fops->func = NULL;
                   ~~~~~~~~^
   include/linux/bpf.h:47:8: note: forward declaration of 'struct ftrace_ops'
   struct ftrace_ops;
          ^
   kernel/bpf/trampoline.c:416:11: error: incomplete definition of type 'struct ftrace_ops'
                   tr->fops->trampoline = 0;
                   ~~~~~~~~^
   include/linux/bpf.h:47:8: note: forward declaration of 'struct ftrace_ops'
   struct ftrace_ops;
          ^
   kernel/bpf/trampoline.c:431:67: warning: declaration of 'enum ftrace_ops_cmd' will not be visible outside of this function [-Wvisibility]
   static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd)
                                                                     ^
   kernel/bpf/trampoline.c:431:12: error: conflicting types for 'bpf_tramp_ftrace_ops_func'
   static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd)
              ^
   kernel/bpf/trampoline.c:30:12: note: previous declaration is here
   static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *op, enum ftrace_ops_cmd cmd);
              ^
   kernel/bpf/trampoline.c:431:82: error: variable has incomplete type 'enum ftrace_ops_cmd'
   static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd)
                                                                                    ^
   kernel/bpf/trampoline.c:431:67: note: forward declaration of 'enum ftrace_ops_cmd'
   static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd)
                                                                     ^
   kernel/bpf/trampoline.c:433:33: error: incomplete definition of type 'struct ftrace_ops'
           struct bpf_trampoline *tr = ops->private;
                                       ~~~^
   include/linux/bpf.h:47:8: note: forward declaration of 'struct ftrace_ops'
   struct ftrace_ops;
          ^
   kernel/bpf/trampoline.c:448:7: error: use of undeclared identifier 'FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY'
           case FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY:
                ^
   kernel/bpf/trampoline.c:452:7: error: use of undeclared identifier 'FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY'
           case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY:
                ^
   kernel/bpf/trampoline.c:454:11: error: incomplete definition of type 'struct ftrace_ops'
                   tr->fops->flags &= ~FTRACE_OPS_FL_SHARE_IPMODIFY;
                   ~~~~~~~~^
   include/linux/bpf.h:47:8: note: forward declaration of 'struct ftrace_ops'
   struct ftrace_ops;
          ^
   kernel/bpf/trampoline.c:454:23: error: use of undeclared identifier 'FTRACE_OPS_FL_SHARE_IPMODIFY'
                   tr->fops->flags &= ~FTRACE_OPS_FL_SHARE_IPMODIFY;
                                       ^
   2 warnings and 14 errors generated.


vim +30 kernel/bpf/trampoline.c

    29	
  > 30	static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *op, enum ftrace_ops_cmd cmd);
    31
kernel test robot June 2, 2022, 12:08 a.m. UTC | #2 
Hi Song,

I love your patch! Perhaps something to improve:

[auto build test WARNING on bpf-next/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Song-Liu/ftrace-host-klp-and-bpf-trampoline-together/20220602-020112
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
config: i386-randconfig-a003 (https://download.01.org/0day-ci/archive/20220602/202206020707.jsHlBldB-lkp@intel.com/config)
compiler: gcc-11 (Debian 11.3.0-1) 11.3.0
reproduce (this is a W=1 build):
        # https://github.com/intel-lab-lkp/linux/commit/7edcf1c49617641579f2bc36b86c7d59bea20aef
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review Song-Liu/ftrace-host-klp-and-bpf-trampoline-together/20220602-020112
        git checkout 7edcf1c49617641579f2bc36b86c7d59bea20aef
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        make W=1 O=build_dir ARCH=i386 SHELL=/bin/bash kernel/bpf/

If you fix the issue, kindly add following tag where applicable
Reported-by: kernel test robot <lkp@intel.com>

All warnings (new ones prefixed by >>):

>> kernel/bpf/trampoline.c:30:66: warning: 'enum ftrace_ops_cmd' declared inside parameter list will not be visible outside of this definition or declaration
      30 | static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *op, enum ftrace_ops_cmd cmd);
         |                                                                  ^~~~~~~~~~~~~~
   kernel/bpf/trampoline.c: In function 'bpf_trampoline_lookup':
   kernel/bpf/trampoline.c:92:35: error: invalid application of 'sizeof' to incomplete type 'struct ftrace_ops'
      92 |         tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);
         |                                   ^~~~~~
   kernel/bpf/trampoline.c:100:17: error: invalid use of undefined type 'struct ftrace_ops'
     100 |         tr->fops->private = tr;
         |                 ^~
   kernel/bpf/trampoline.c:101:17: error: invalid use of undefined type 'struct ftrace_ops'
     101 |         tr->fops->ops_func = bpf_tramp_ftrace_ops_func;
         |                 ^~
   kernel/bpf/trampoline.c: In function 'bpf_trampoline_update':
   kernel/bpf/trampoline.c:397:25: error: invalid use of undefined type 'struct ftrace_ops'
     397 |                 tr->fops->flags |= FTRACE_OPS_FL_SHARE_IPMODIFY;
         |                         ^~
   kernel/bpf/trampoline.c:397:36: error: 'FTRACE_OPS_FL_SHARE_IPMODIFY' undeclared (first use in this function)
     397 |                 tr->fops->flags |= FTRACE_OPS_FL_SHARE_IPMODIFY;
         |                                    ^~~~~~~~~~~~~~~~~~~~~~~~~~~~
   kernel/bpf/trampoline.c:397:36: note: each undeclared identifier is reported only once for each function it appears in
   kernel/bpf/trampoline.c:415:25: error: invalid use of undefined type 'struct ftrace_ops'
     415 |                 tr->fops->func = NULL;
         |                         ^~
   kernel/bpf/trampoline.c:416:25: error: invalid use of undefined type 'struct ftrace_ops'
     416 |                 tr->fops->trampoline = 0;
         |                         ^~
   kernel/bpf/trampoline.c: At top level:
   kernel/bpf/trampoline.c:431:67: warning: 'enum ftrace_ops_cmd' declared inside parameter list will not be visible outside of this definition or declaration
     431 | static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd)
         |                                                                   ^~~~~~~~~~~~~~
   kernel/bpf/trampoline.c:431:82: error: parameter 2 ('cmd') has incomplete type
     431 | static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd)
         |                                                              ~~~~~~~~~~~~~~~~~~~~^~~
   kernel/bpf/trampoline.c: In function 'bpf_tramp_ftrace_ops_func':
   kernel/bpf/trampoline.c:433:40: error: invalid use of undefined type 'struct ftrace_ops'
     433 |         struct bpf_trampoline *tr = ops->private;
         |                                        ^~
   kernel/bpf/trampoline.c:448:14: error: 'FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY' undeclared (first use in this function)
     448 |         case FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY:
         |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   kernel/bpf/trampoline.c:452:14: error: 'FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY' undeclared (first use in this function)
     452 |         case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY:
         |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   kernel/bpf/trampoline.c:454:25: error: invalid use of undefined type 'struct ftrace_ops'
     454 |                 tr->fops->flags &= ~FTRACE_OPS_FL_SHARE_IPMODIFY;
         |                         ^~
   kernel/bpf/trampoline.c:454:37: error: 'FTRACE_OPS_FL_SHARE_IPMODIFY' undeclared (first use in this function)
     454 |                 tr->fops->flags &= ~FTRACE_OPS_FL_SHARE_IPMODIFY;
         |                                     ^~~~~~~~~~~~~~~~~~~~~~~~~~~~


vim +30 kernel/bpf/trampoline.c

    29	
  > 30	static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *op, enum ftrace_ops_cmd cmd);
    31
kernel test robot June 2, 2022, 1:30 a.m. UTC | #3 
Hi Song,

I love your patch! Yet something to improve:

[auto build test ERROR on bpf-next/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Song-Liu/ftrace-host-klp-and-bpf-trampoline-together/20220602-020112
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
config: x86_64-randconfig-a006 (https://download.01.org/0day-ci/archive/20220602/202206020957.KETjl2xP-lkp@intel.com/config)
compiler: gcc-11 (Debian 11.3.0-1) 11.3.0
reproduce (this is a W=1 build):
        # https://github.com/intel-lab-lkp/linux/commit/7edcf1c49617641579f2bc36b86c7d59bea20aef
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review Song-Liu/ftrace-host-klp-and-bpf-trampoline-together/20220602-020112
        git checkout 7edcf1c49617641579f2bc36b86c7d59bea20aef
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        make W=1 O=build_dir ARCH=x86_64 SHELL=/bin/bash

If you fix the issue, kindly add following tag where applicable
Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

   kernel/bpf/trampoline.c: In function 'bpf_trampoline_lookup':
>> kernel/bpf/trampoline.c:101:17: error: 'struct ftrace_ops' has no member named 'ops_func'
     101 |         tr->fops->ops_func = bpf_tramp_ftrace_ops_func;
         |                 ^~
   kernel/bpf/trampoline.c: In function 'bpf_trampoline_update':
>> kernel/bpf/trampoline.c:416:25: error: 'struct ftrace_ops' has no member named 'trampoline'
     416 |                 tr->fops->trampoline = 0;
         |                         ^~


vim +101 kernel/bpf/trampoline.c

    74	
    75	static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
    76	{
    77		struct bpf_trampoline *tr;
    78		struct hlist_head *head;
    79		int i;
    80	
    81		mutex_lock(&trampoline_mutex);
    82		head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)];
    83		hlist_for_each_entry(tr, head, hlist) {
    84			if (tr->key == key) {
    85				refcount_inc(&tr->refcnt);
    86				goto out;
    87			}
    88		}
    89		tr = kzalloc(sizeof(*tr), GFP_KERNEL);
    90		if (!tr)
    91			goto out;
    92		tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);
    93		if (!tr->fops) {
    94			kfree(tr);
    95			tr = NULL;
    96			goto out;
    97		}
    98	
    99		tr->key = key;
   100		tr->fops->private = tr;
 > 101		tr->fops->ops_func = bpf_tramp_ftrace_ops_func;
   102		INIT_HLIST_NODE(&tr->hlist);
   103		hlist_add_head(&tr->hlist, head);
   104		refcount_set(&tr->refcnt, 1);
   105		mutex_init(&tr->mutex);
   106		for (i = 0; i < BPF_TRAMP_MAX; i++)
   107			INIT_HLIST_HEAD(&tr->progs_hlist[i]);
   108	out:
   109		mutex_unlock(&trampoline_mutex);
   110		return tr;
   111	}
   112
diff mbox series

Patch

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index a6e06f384e81..20a8ed600ca6 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -44,6 +44,7 @@  struct kobject;
 struct mem_cgroup;
 struct module;
 struct bpf_func_state;
+struct ftrace_ops;
 
 extern struct idr btf_idr;
 extern spinlock_t btf_idr_lock;
@@ -816,6 +817,7 @@  struct bpf_tramp_image {
 struct bpf_trampoline {
 	/* hlist for trampoline_table */
 	struct hlist_node hlist;
+	struct ftrace_ops *fops;
 	/* serializes access to fields of this trampoline */
 	struct mutex mutex;
 	refcount_t refcnt;
@@ -838,6 +840,7 @@  struct bpf_trampoline {
 	struct bpf_tramp_image *cur_image;
 	u64 selector;
 	struct module *mod;
+	bool indirect_call;
 };
 
 struct bpf_attach_target_info {
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 93c7675f0c9e..33d70d6ed165 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -27,6 +27,8 @@  static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE];
 /* serializes access to trampoline_table */
 static DEFINE_MUTEX(trampoline_mutex);
 
+static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *op, enum ftrace_ops_cmd cmd);
+
 bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
 {
 	enum bpf_attach_type eatype = prog->expected_attach_type;
@@ -87,8 +89,16 @@  static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
 	tr = kzalloc(sizeof(*tr), GFP_KERNEL);
 	if (!tr)
 		goto out;
+	tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);
+	if (!tr->fops) {
+		kfree(tr);
+		tr = NULL;
+		goto out;
+	}
 
 	tr->key = key;
+	tr->fops->private = tr;
+	tr->fops->ops_func = bpf_tramp_ftrace_ops_func;
 	INIT_HLIST_NODE(&tr->hlist);
 	hlist_add_head(&tr->hlist, head);
 	refcount_set(&tr->refcnt, 1);
@@ -126,7 +136,7 @@  static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)
 	int ret;
 
 	if (tr->func.ftrace_managed)
-		ret = unregister_ftrace_direct((long)ip, (long)old_addr);
+		ret = unregister_ftrace_direct_multi(tr->fops, (long)old_addr);
 	else
 		ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL);
 
@@ -135,15 +145,20 @@  static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)
 	return ret;
 }
 
-static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr)
+static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr,
+			 bool lock_direct_mutex)
 {
 	void *ip = tr->func.addr;
 	int ret;
 
-	if (tr->func.ftrace_managed)
-		ret = modify_ftrace_direct((long)ip, (long)old_addr, (long)new_addr);
-	else
+	if (tr->func.ftrace_managed) {
+		if (lock_direct_mutex)
+			ret = modify_ftrace_direct_multi(tr->fops, (long)new_addr);
+		else
+			ret = modify_ftrace_direct_multi_nolock(tr->fops, (long)new_addr);
+	} else {
 		ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr);
+	}
 	return ret;
 }
 
@@ -161,10 +176,15 @@  static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
 	if (bpf_trampoline_module_get(tr))
 		return -ENOENT;
 
-	if (tr->func.ftrace_managed)
-		ret = register_ftrace_direct((long)ip, (long)new_addr);
-	else
+	if (tr->func.ftrace_managed) {
+		ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 0);
+		ret = register_ftrace_direct_multi(tr->fops, (long)new_addr);
+		if (ret)
+			ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 1, 0);
+
+	} else {
 		ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr);
+	}
 
 	if (ret)
 		bpf_trampoline_module_put(tr);
@@ -330,7 +350,7 @@  static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx)
 	return ERR_PTR(err);
 }
 
-static int bpf_trampoline_update(struct bpf_trampoline *tr)
+static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex)
 {
 	struct bpf_tramp_image *im;
 	struct bpf_tramp_links *tlinks;
@@ -363,20 +383,40 @@  static int bpf_trampoline_update(struct bpf_trampoline *tr)
 	if (ip_arg)
 		flags |= BPF_TRAMP_F_IP_ARG;
 
+again:
+	if (tr->indirect_call)
+		flags |= BPF_TRAMP_F_ORIG_STACK;
+
 	err = arch_prepare_bpf_trampoline(im, im->image, im->image + PAGE_SIZE,
 					  &tr->func.model, flags, tlinks,
 					  tr->func.addr);
 	if (err < 0)
 		goto out;
 
+	if (tr->indirect_call)
+		tr->fops->flags |= FTRACE_OPS_FL_SHARE_IPMODIFY;
+
 	WARN_ON(tr->cur_image && tr->selector == 0);
 	WARN_ON(!tr->cur_image && tr->selector);
 	if (tr->cur_image)
 		/* progs already running at this address */
-		err = modify_fentry(tr, tr->cur_image->image, im->image);
+		err = modify_fentry(tr, tr->cur_image->image, im->image, lock_direct_mutex);
 	else
 		/* first time registering */
 		err = register_fentry(tr, im->image);
+
+	if (err == -EAGAIN) {
+		if (WARN_ON_ONCE(tr->indirect_call))
+			goto out;
+		/* should only retry on the first register */
+		if (WARN_ON_ONCE(tr->cur_image))
+			goto out;
+		tr->indirect_call = true;
+		tr->fops->func = NULL;
+		tr->fops->trampoline = 0;
+		goto again;
+	}
+
 	if (err)
 		goto out;
 	if (tr->cur_image)
@@ -388,6 +428,41 @@  static int bpf_trampoline_update(struct bpf_trampoline *tr)
 	return err;
 }
 
+static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd)
+{
+	struct bpf_trampoline *tr = ops->private;
+	int ret;
+
+	/*
+	 * The normal locking order is
+	 *    tr->mutex => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c)
+	 *
+	 * This is called from prepare_direct_functions_for_ipmodify, with
+	 * direct_mutex locked. Use mutex_trylock() to avoid dead lock.
+	 * Also, bpf_trampoline_update here should not lock direct_mutex.
+	 */
+	if (!mutex_trylock(&tr->mutex))
+		return -EAGAIN;
+
+	switch (cmd) {
+	case FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY:
+		tr->indirect_call = true;
+		ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
+		break;
+	case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY:
+		tr->indirect_call = false;
+		tr->fops->flags &= ~FTRACE_OPS_FL_SHARE_IPMODIFY;
+		ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	};
+	mutex_unlock(&tr->mutex);
+	return ret;
+}
+
+
 static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
 {
 	switch (prog->expected_attach_type) {
@@ -460,7 +535,7 @@  int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline
 
 	hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]);
 	tr->progs_cnt[kind]++;
-	err = bpf_trampoline_update(tr);
+	err = bpf_trampoline_update(tr, true /* lock_direct_mutex */);
 	if (err) {
 		hlist_del_init(&link->tramp_hlist);
 		tr->progs_cnt[kind]--;
@@ -487,7 +562,7 @@  int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampolin
 	}
 	hlist_del_init(&link->tramp_hlist);
 	tr->progs_cnt[kind]--;
-	err = bpf_trampoline_update(tr);
+	err = bpf_trampoline_update(tr, true /* lock_direct_mutex */);
 out:
 	mutex_unlock(&tr->mutex);
 	return err;
@@ -535,6 +610,7 @@  void bpf_trampoline_put(struct bpf_trampoline *tr)
 	 * multiple rcu callbacks.
 	 */
 	hlist_del(&tr->hlist);
+	kfree(tr->fops);
 	kfree(tr);
 out:
 	mutex_unlock(&trampoline_mutex);