[bpf-next,v4,2/4] bpf: add bpf_cpu_cycles_to_ns helper

Commit Message

Vadim Fedorenko Nov. 7, 2024, 9:12 p.m. UTC

The new helper should be used to convert cycles received by
bpf_get_cpu_cycle() into nanoseconds.

Signed-off-by: Vadim Fedorenko <[e-mail]>

v4:
* change helper name to bpf_cpu_cycles_to_ns.
* hide it behind CONFIG_GENERIC_GETTIMEOFDAY to avoid exposing on
  unsupported architectures.
---
 arch/x86/net/bpf_jit_comp.c   | 22 ++++++++++++++++++++++
 arch/x86/net/bpf_jit_comp32.c | 19 +++++++++++++++++++
 include/linux/bpf.h           |  1 +
 kernel/bpf/helpers.c          | 10 +++++++++-
 4 files changed, 51 insertions(+), 1 deletion(-)

Comments

kernel test robot Nov. 8, 2024, 1:54 a.m. UTC | #1

Hi Vadim,

kernel test robot noticed the following build errors:

[auto build test ERROR on bpf-next/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Vadim-Fedorenko/bpf-add-bpf_cpu_cycles_to_ns-helper/20241108-051950
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
patch link:    https://lore.kernel.org/r/20241107211206.2814069-2-vadfed%40meta.com
patch subject: [PATCH bpf-next v4 2/4] bpf: add bpf_cpu_cycles_to_ns helper
config: sh-allmodconfig (https://download.01.org/0day-ci/archive/20241108/[e-mail]/config)
compiler: sh4-linux-gcc (GCC) 14.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20241108/[e-mail]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[e-mail]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[e-mail]/

All errors (new ones prefixed by >>):

   kernel/bpf/helpers.c:3030:18: warning: extra tokens at end of #ifdef directive
    3030 | #ifdef IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)
         |                  ^
   kernel/bpf/helpers.c: In function 'bpf_get_cpu_cycles':
   kernel/bpf/helpers.c:3033:16: error: implicit declaration of function '__arch_get_hw_counter' [-Wimplicit-function-declaration]
    3033 |         return __arch_get_hw_counter(1, NULL);
         |                ^~~~~~~~~~~~~~~~~~~~~
   kernel/bpf/helpers.c: In function 'bpf_cpu_cycles_to_ns':
   kernel/bpf/helpers.c:3038:38: error: implicit declaration of function '__arch_get_k_vdso_data' [-Wimplicit-function-declaration]
    3038 |         const struct vdso_data *vd = __arch_get_k_vdso_data();
         |                                      ^~~~~~~~~~~~~~~~~~~~~~
>> kernel/bpf/helpers.c:3038:38: error: initialization of 'const struct vdso_data *' from 'int' makes pointer from integer without a cast [-Wint-conversion]
   kernel/bpf/helpers.c:3040:42: error: invalid use of undefined type 'const struct vdso_data'
    3040 |         return mul_u64_u32_shr(cycles, vd->mult, vd->shift);
         |                                          ^~
   kernel/bpf/helpers.c:3040:52: error: invalid use of undefined type 'const struct vdso_data'
    3040 |         return mul_u64_u32_shr(cycles, vd->mult, vd->shift);
         |                                                    ^~
   kernel/bpf/helpers.c: At top level:
   kernel/bpf/helpers.c:3135:18: warning: extra tokens at end of #ifdef directive
    3135 | #ifdef IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)
         |                  ^
   kernel/bpf/helpers.c: In function 'bpf_cpu_cycles_to_ns':
   kernel/bpf/helpers.c:3041:1: warning: control reaches end of non-void function [-Wreturn-type]
    3041 | }
         | ^


vim +3038 kernel/bpf/helpers.c

  3035	
  3036	__bpf_kfunc u64 bpf_cpu_cycles_to_ns(u64 cycles)
  3037	{
> 3038		const struct vdso_data *vd = __arch_get_k_vdso_data();
  3039	
  3040		return mul_u64_u32_shr(cycles, vd->mult, vd->shift);
  3041	}
  3042	#endif
  3043	__bpf_kfunc_end_defs();
  3044

Patch

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 4f78ed93ee7f..ddc73d9a90f4 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -11,6 +11,7 @@ 
 #include <linux/bpf.h>
 #include <linux/memory.h>
 #include <linux/sort.h>
+#include <linux/clocksource.h>
 #include <asm/extable.h>
 #include <asm/ftrace.h>
 #include <asm/set_memory.h>
@@ -2146,6 +2147,24 @@  st:			if (is_imm8(insn->off))
 				break;
 			}
 
+			if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
+			    imm32 == BPF_CALL_IMM(bpf_cpu_cycles_to_ns) &&
+			    boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
+				u32 mult, shift;
+
+				clocks_calc_mult_shift(&mult, &shift, tsc_khz, USEC_PER_SEC, 0);
+				/* imul RAX, RDI, mult */
+				maybe_emit_mod(&prog, BPF_REG_1, BPF_REG_0, true);
+				EMIT2_off32(0x69, add_2reg(0xC0, BPF_REG_1, BPF_REG_0),
+					    mult);
+
+				/* shr RAX, shift (which is less than 64) */
+				maybe_emit_1mod(&prog, BPF_REG_0, true);
+				EMIT3(0xC1, add_1reg(0xE8, BPF_REG_0), shift);
+
+				break;
+			}
+
 			func = (u8 *) __bpf_call_base + imm32;
 			if (tail_call_reachable) {
 				LOAD_TAIL_CALL_CNT_PTR(bpf_prog->aux->stack_depth);
@@ -3678,5 +3697,8 @@  bool bpf_jit_inlines_kfunc_call(s32 imm)
 {
 	if (imm == BPF_CALL_IMM(bpf_get_cpu_cycles))
 		return true;
+	if (imm == BPF_CALL_IMM(bpf_cpu_cycles_to_ns) &&
+	    boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
+		return true;
 	return false;
 }
diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
index e6097a371b69..34f762f28c82 100644
--- a/arch/x86/net/bpf_jit_comp32.c
+++ b/arch/x86/net/bpf_jit_comp32.c
@@ -12,6 +12,7 @@ 
 #include <linux/netdevice.h>
 #include <linux/filter.h>
 #include <linux/if_vlan.h>
+#include <linux/clocksource.h>
 #include <asm/cacheflush.h>
 #include <asm/set_memory.h>
 #include <asm/nospec-branch.h>
@@ -2100,6 +2101,24 @@  static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 					EMIT2(0x0F, 0x31);
 					break;
 				}
+				if (imm32 == BPF_CALL_IMM(bpf_cpu_cycles_to_ns) &&
+				    boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
+					u32 mult, shift;
+
+					clocks_calc_mult_shift(&mult, &shift, tsc_khz,
+							       USEC_PER_SEC, 0);
+
+					/* move parameter to BPF_REG_0 */
+					emit_ia32_mov_r64(true, bpf2ia32[BPF_REG_0],
+							  bpf2ia32[BPF_REG_1], true, true,
+							  &prog, bpf_prog->aux);
+					/* multiply parameter by mut */
+					emit_ia32_mul_i64(bpf2ia32[BPF_REG_0],
+							  mult, true, &prog);
+					/* shift parameter by shift which is less than 64 */
+					emit_ia32_rsh_i64(bpf2ia32[BPF_REG_0],
+							  shift, true, &prog);
+				}
 
 				err = emit_kfunc_call(bpf_prog,
 						      image + addrs[i],
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index fed5f36d387a..46fa662d95e4 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -3331,6 +3331,7 @@  u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
 /* Inlined kfuncs */
 #if IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)
 u64 bpf_get_cpu_cycles(void);
+u64 bpf_cpu_cycles_to_ns(u64 cycles);
 #endif
 
 #if defined(CONFIG_NET)
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index c07cb058e710..d794ddc74de5 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -25,6 +25,7 @@ 
 #include <linux/kasan.h>
 #if IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)
 #include <vdso/datapage.h>
+#include <asm/vdso/vsyscall.h>
 #endif
 
 #include "../../lib/kstrtox.h"
@@ -3031,8 +3032,14 @@  __bpf_kfunc u64 bpf_get_cpu_cycles(void)
 {
 	return __arch_get_hw_counter(1, NULL);
 }
-#endif
 
+__bpf_kfunc u64 bpf_cpu_cycles_to_ns(u64 cycles)
+{
+	const struct vdso_data *vd = __arch_get_k_vdso_data();
+
+	return mul_u64_u32_shr(cycles, vd->mult, vd->shift);
+}
+#endif
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(generic_btf_ids)
@@ -3127,6 +3134,7 @@  BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLE
 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
 #ifdef IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)
 BTF_ID_FLAGS(func, bpf_get_cpu_cycles, KF_FASTCALL)
+BTF_ID_FLAGS(func, bpf_cpu_cycles_to_ns, KF_FASTCALL)
 #endif
 BTF_KFUNCS_END(common_btf_ids)