| Message ID | 20240808-tracepoint-v6-4-a23f800f1189@google.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Tracepoints and static branch in Rust | expand |
On Thu, Aug 08, 2024 at 05:23:40PM +0000, Alice Ryhl wrote: > To avoid duplication of inline asm between C and Rust, we need to > import the inline asm from the relevant `jump_label.h` header into Rust. > To make that easier, this patch updates the header files to expose the > inline asm via a new ARCH_STATIC_BRANCH_ASM macro. > > The header files are all updated to define a ARCH_STATIC_BRANCH_ASM that > takes the same arguments in a consistent order so that Rust can use the > same logic for every architecture. > > Link: https://lore.kernel.org/r/20240725183325.122827-7-ojeda@kernel.org [1] > Signed-off-by: Alice Ryhl <aliceryhl@google.com> I can't help but notice your risc-v conversion is much nicer than the one you did for ARM. But I'm not going to complain too loudly about that. It's a shame about that 2 for JUMP_LABEL_HACK, having to lift that nonsense all the way to Rust, but alas. This will do, thanks! Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> > --- > arch/arm/include/asm/jump_label.h | 14 +++++---- > arch/arm64/include/asm/jump_label.h | 20 ++++++++----- > arch/loongarch/include/asm/jump_label.h | 16 +++++++---- > arch/riscv/include/asm/jump_label.h | 50 ++++++++++++++++++--------------- > arch/x86/include/asm/jump_label.h | 38 ++++++++++--------------- > 5 files changed, 75 insertions(+), 63 deletions(-) > > diff --git a/arch/arm/include/asm/jump_label.h b/arch/arm/include/asm/jump_label.h > index e4eb54f6cd9f..a35aba7f548c 100644 > --- a/arch/arm/include/asm/jump_label.h > +++ b/arch/arm/include/asm/jump_label.h > @@ -9,13 +9,17 @@ > > #define JUMP_LABEL_NOP_SIZE 4 > > +/* This macro is also expanded on the Rust side. */ > +#define ARCH_STATIC_BRANCH_ASM(key, label) \ > + "1:\n\t" \ > + WASM(nop) "\n\t" \ > + ".pushsection __jump_table, \"aw\"\n\t" \ > + ".word 1b, " label ", " key "\n\t" \ > + ".popsection\n\t" \ > + > static __always_inline bool arch_static_branch(struct static_key *key, bool branch) > { > - asm goto("1:\n\t" > - WASM(nop) "\n\t" > - ".pushsection __jump_table, \"aw\"\n\t" > - ".word 1b, %l[l_yes], %c0\n\t" > - ".popsection\n\t" > + asm goto(ARCH_STATIC_BRANCH_ASM("%c0", "%l[l_yes]") > : : "i" (&((char *)key)[branch]) : : l_yes); > > return false; > diff --git a/arch/arm64/include/asm/jump_label.h b/arch/arm64/include/asm/jump_label.h > index a0a5bbae7229..424ed421cd97 100644 > --- a/arch/arm64/include/asm/jump_label.h > +++ b/arch/arm64/include/asm/jump_label.h > @@ -19,10 +19,14 @@ > #define JUMP_TABLE_ENTRY(key, label) \ > ".pushsection __jump_table, \"aw\"\n\t" \ > ".align 3\n\t" \ > - ".long 1b - ., %l["#label"] - .\n\t" \ > - ".quad %c0 - .\n\t" \ > - ".popsection\n\t" \ > - : : "i"(key) : : label > + ".long 1b - ., " label " - .\n\t" \ > + ".quad " key " - .\n\t" \ > + ".popsection\n\t" > + > +/* This macro is also expanded on the Rust side. */ > +#define ARCH_STATIC_BRANCH_ASM(key, label) \ > + "1: nop\n\t" \ > + JUMP_TABLE_ENTRY(key, label) > > static __always_inline bool arch_static_branch(struct static_key * const key, > const bool branch) > @@ -30,8 +34,8 @@ static __always_inline bool arch_static_branch(struct static_key * const key, > char *k = &((char *)key)[branch]; > > asm goto( > - "1: nop \n\t" > - JUMP_TABLE_ENTRY(k, l_yes) > + ARCH_STATIC_BRANCH_ASM("%c0", "%l[l_yes]") > + : : "i"(k) : : l_yes > ); > > return false; > @@ -43,9 +47,11 @@ static __always_inline bool arch_static_branch_jump(struct static_key * const ke > const bool branch) > { > char *k = &((char *)key)[branch]; > + > asm goto( > "1: b %l[l_yes] \n\t" > - JUMP_TABLE_ENTRY(k, l_yes) > + JUMP_TABLE_ENTRY("%c0", "%l[l_yes]") > + : : "i"(k) : : l_yes > ); > return false; > l_yes: > diff --git a/arch/loongarch/include/asm/jump_label.h b/arch/loongarch/include/asm/jump_label.h > index 29acfe3de3fa..8a924bd69d19 100644 > --- a/arch/loongarch/include/asm/jump_label.h > +++ b/arch/loongarch/include/asm/jump_label.h > @@ -13,18 +13,22 @@ > > #define JUMP_LABEL_NOP_SIZE 4 > > -#define JUMP_TABLE_ENTRY \ > +/* This macro is also expanded on the Rust side. */ > +#define JUMP_TABLE_ENTRY(key, label) \ > ".pushsection __jump_table, \"aw\" \n\t" \ > ".align 3 \n\t" \ > - ".long 1b - ., %l[l_yes] - . \n\t" \ > - ".quad %0 - . \n\t" \ > + ".long 1b - ., " label " - . \n\t" \ > + ".quad " key " - . \n\t" \ > ".popsection \n\t" > > +#define ARCH_STATIC_BRANCH_ASM(key, label) \ > + "1: nop \n\t" \ > + JUMP_TABLE_ENTRY(key, label) > + > static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch) > { > asm goto( > - "1: nop \n\t" > - JUMP_TABLE_ENTRY > + ARCH_STATIC_BRANCH_ASM("%0", "%l[l_yes]") > : : "i"(&((char *)key)[branch]) : : l_yes); > > return false; > @@ -37,7 +41,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key * const ke > { > asm goto( > "1: b %l[l_yes] \n\t" > - JUMP_TABLE_ENTRY > + JUMP_TABLE_ENTRY("%0", "%l[l_yes]") > : : "i"(&((char *)key)[branch]) : : l_yes); > > return false; > diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h > index 1c768d02bd0c..f6342d456372 100644 > --- a/arch/riscv/include/asm/jump_label.h > +++ b/arch/riscv/include/asm/jump_label.h > @@ -16,21 +16,28 @@ > > #define JUMP_LABEL_NOP_SIZE 4 > > +#define JUMP_TABLE_ENTRY(key, label) \ > + ".pushsection __jump_table, \"aw\" \n\t" \ > + ".align " RISCV_LGPTR " \n\t" \ > + ".long 1b - ., " label " - . \n\t" \ > + "" RISCV_PTR " " key " - . \n\t" \ > + ".popsection \n\t" > + > +/* This macro is also expanded on the Rust side. */ > +#define ARCH_STATIC_BRANCH_ASM(key, label) \ > + " .align 2 \n\t" \ > + " .option push \n\t" \ > + " .option norelax \n\t" \ > + " .option norvc \n\t" \ > + "1: j " label " \n\t" \ > + " .option pop \n\t" \ > + JUMP_TABLE_ENTRY(key, label) > + > static __always_inline bool arch_static_branch(struct static_key * const key, > const bool branch) > { > asm goto( > - " .align 2 \n\t" > - " .option push \n\t" > - " .option norelax \n\t" > - " .option norvc \n\t" > - "1: nop \n\t" > - " .option pop \n\t" > - " .pushsection __jump_table, \"aw\" \n\t" > - " .align " RISCV_LGPTR " \n\t" > - " .long 1b - ., %l[label] - . \n\t" > - " " RISCV_PTR " %0 - . \n\t" > - " .popsection \n\t" > + ARCH_STATIC_BRANCH_ASM("%0", "%l[label]") > : : "i"(&((char *)key)[branch]) : : label); > > return false; > @@ -38,21 +45,20 @@ static __always_inline bool arch_static_branch(struct static_key * const key, > return true; > } > > +#define ARCH_STATIC_BRANCH_JUMP_ASM(key, label) \ > + " .align 2 \n\t" \ > + " .option push \n\t" \ > + " .option norelax \n\t" \ > + " .option norvc \n\t" \ > + "1: nop \n\t" \ > + " .option pop \n\t" \ > + JUMP_TABLE_ENTRY(key, label) > + > static __always_inline bool arch_static_branch_jump(struct static_key * const key, > const bool branch) > { > asm goto( > - " .align 2 \n\t" > - " .option push \n\t" > - " .option norelax \n\t" > - " .option norvc \n\t" > - "1: j %l[label] \n\t" > - " .option pop \n\t" > - " .pushsection __jump_table, \"aw\" \n\t" > - " .align " RISCV_LGPTR " \n\t" > - " .long 1b - ., %l[label] - . \n\t" > - " " RISCV_PTR " %0 - . \n\t" > - " .popsection \n\t" > + ARCH_STATIC_BRANCH_JUMP_ASM("%0", "%l[label]") > : : "i"(&((char *)key)[branch]) : : label); > > return false; > diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h > index cbbef32517f0..87a3a0f5bd22 100644 > --- a/arch/x86/include/asm/jump_label.h > +++ b/arch/x86/include/asm/jump_label.h > @@ -12,49 +12,41 @@ > #include <linux/stringify.h> > #include <linux/types.h> > > -#define JUMP_TABLE_ENTRY \ > +#define JUMP_TABLE_ENTRY(key, label) \ > ".pushsection __jump_table, \"aw\" \n\t" \ > _ASM_ALIGN "\n\t" \ > ".long 1b - . \n\t" \ > - ".long %l[l_yes] - . \n\t" \ > - _ASM_PTR "%c0 + %c1 - .\n\t" \ > + ".long " label "- . \n\t" \ > + _ASM_PTR " " key " - . \n\t" \ > ".popsection \n\t" > > +/* This macro is also expanded on the Rust side. */ > #ifdef CONFIG_HAVE_JUMP_LABEL_HACK > - > -static __always_inline bool arch_static_branch(struct static_key *key, bool branch) > -{ > - asm goto("1:" > - "jmp %l[l_yes] # objtool NOPs this \n\t" > - JUMP_TABLE_ENTRY > - : : "i" (key), "i" (2 | branch) : : l_yes); > - > - return false; > -l_yes: > - return true; > -} > - > +#define ARCH_STATIC_BRANCH_ASM(key, label) \ > + "1: jmp " label " # objtool NOPs this \n\t" \ > + JUMP_TABLE_ENTRY(key, label) > #else /* !CONFIG_HAVE_JUMP_LABEL_HACK */ > +#define ARCH_STATIC_BRANCH_ASM(key, label) \ > + "1: .byte " __stringify(BYTES_NOP5) "\n\t" \ > + JUMP_TABLE_ENTRY(key, label) > +#endif /* CONFIG_HAVE_JUMP_LABEL_HACK */ > > static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch) > { > - asm goto("1:" > - ".byte " __stringify(BYTES_NOP5) "\n\t" > - JUMP_TABLE_ENTRY > - : : "i" (key), "i" (branch) : : l_yes); > + int hack_bit = IS_ENABLED(CONFIG_HAVE_JUMP_LABEL_HACK) ? 2 : 0; > + asm goto(ARCH_STATIC_BRANCH_ASM("%c0 + %c1", "%l[l_yes]") > + : : "i" (key), "i" (hack_bit | branch) : : l_yes); > > return false; > l_yes: > return true; > } > > -#endif /* CONFIG_HAVE_JUMP_LABEL_HACK */ > - > static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch) > { > asm goto("1:" > "jmp %l[l_yes]\n\t" > - JUMP_TABLE_ENTRY > + JUMP_TABLE_ENTRY("%c0 + %c1", "%l[l_yes]") > : : "i" (key), "i" (branch) : : l_yes); > > return false; > > -- > 2.46.0.76.ge559c4bf1a-goog >
On Thu, Aug 8, 2024 at 7:23 PM Alice Ryhl <aliceryhl@google.com> wrote: > > To avoid duplication of inline asm between C and Rust, we need to > import the inline asm from the relevant `jump_label.h` header into Rust. > To make that easier, this patch updates the header files to expose the > inline asm via a new ARCH_STATIC_BRANCH_ASM macro. > > The header files are all updated to define a ARCH_STATIC_BRANCH_ASM that > takes the same arguments in a consistent order so that Rust can use the > same logic for every architecture. > > Link: https://lore.kernel.org/r/20240725183325.122827-7-ojeda@kernel.org [1] This link is in the wrong place. It's supposed to be mentioned as a dependency for this series. Also, I intended to have the same tags here as I did on the last patch. Alice
diff --git a/arch/arm/include/asm/jump_label.h b/arch/arm/include/asm/jump_label.h index e4eb54f6cd9f..a35aba7f548c 100644 --- a/arch/arm/include/asm/jump_label.h +++ b/arch/arm/include/asm/jump_label.h @@ -9,13 +9,17 @@ #define JUMP_LABEL_NOP_SIZE 4 +/* This macro is also expanded on the Rust side. */ +#define ARCH_STATIC_BRANCH_ASM(key, label) \ + "1:\n\t" \ + WASM(nop) "\n\t" \ + ".pushsection __jump_table, \"aw\"\n\t" \ + ".word 1b, " label ", " key "\n\t" \ + ".popsection\n\t" \ + static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { - asm goto("1:\n\t" - WASM(nop) "\n\t" - ".pushsection __jump_table, \"aw\"\n\t" - ".word 1b, %l[l_yes], %c0\n\t" - ".popsection\n\t" + asm goto(ARCH_STATIC_BRANCH_ASM("%c0", "%l[l_yes]") : : "i" (&((char *)key)[branch]) : : l_yes); return false; diff --git a/arch/arm64/include/asm/jump_label.h b/arch/arm64/include/asm/jump_label.h index a0a5bbae7229..424ed421cd97 100644 --- a/arch/arm64/include/asm/jump_label.h +++ b/arch/arm64/include/asm/jump_label.h @@ -19,10 +19,14 @@ #define JUMP_TABLE_ENTRY(key, label) \ ".pushsection __jump_table, \"aw\"\n\t" \ ".align 3\n\t" \ - ".long 1b - ., %l["#label"] - .\n\t" \ - ".quad %c0 - .\n\t" \ - ".popsection\n\t" \ - : : "i"(key) : : label + ".long 1b - ., " label " - .\n\t" \ + ".quad " key " - .\n\t" \ + ".popsection\n\t" + +/* This macro is also expanded on the Rust side. */ +#define ARCH_STATIC_BRANCH_ASM(key, label) \ + "1: nop\n\t" \ + JUMP_TABLE_ENTRY(key, label) static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch) @@ -30,8 +34,8 @@ static __always_inline bool arch_static_branch(struct static_key * const key, char *k = &((char *)key)[branch]; asm goto( - "1: nop \n\t" - JUMP_TABLE_ENTRY(k, l_yes) + ARCH_STATIC_BRANCH_ASM("%c0", "%l[l_yes]") + : : "i"(k) : : l_yes ); return false; @@ -43,9 +47,11 @@ static __always_inline bool arch_static_branch_jump(struct static_key * const ke const bool branch) { char *k = &((char *)key)[branch]; + asm goto( "1: b %l[l_yes] \n\t" - JUMP_TABLE_ENTRY(k, l_yes) + JUMP_TABLE_ENTRY("%c0", "%l[l_yes]") + : : "i"(k) : : l_yes ); return false; l_yes: diff --git a/arch/loongarch/include/asm/jump_label.h b/arch/loongarch/include/asm/jump_label.h index 29acfe3de3fa..8a924bd69d19 100644 --- a/arch/loongarch/include/asm/jump_label.h +++ b/arch/loongarch/include/asm/jump_label.h @@ -13,18 +13,22 @@ #define JUMP_LABEL_NOP_SIZE 4 -#define JUMP_TABLE_ENTRY \ +/* This macro is also expanded on the Rust side. */ +#define JUMP_TABLE_ENTRY(key, label) \ ".pushsection __jump_table, \"aw\" \n\t" \ ".align 3 \n\t" \ - ".long 1b - ., %l[l_yes] - . \n\t" \ - ".quad %0 - . \n\t" \ + ".long 1b - ., " label " - . \n\t" \ + ".quad " key " - . \n\t" \ ".popsection \n\t" +#define ARCH_STATIC_BRANCH_ASM(key, label) \ + "1: nop \n\t" \ + JUMP_TABLE_ENTRY(key, label) + static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch) { asm goto( - "1: nop \n\t" - JUMP_TABLE_ENTRY + ARCH_STATIC_BRANCH_ASM("%0", "%l[l_yes]") : : "i"(&((char *)key)[branch]) : : l_yes); return false; @@ -37,7 +41,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key * const ke { asm goto( "1: b %l[l_yes] \n\t" - JUMP_TABLE_ENTRY + JUMP_TABLE_ENTRY("%0", "%l[l_yes]") : : "i"(&((char *)key)[branch]) : : l_yes); return false; diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h index 1c768d02bd0c..f6342d456372 100644 --- a/arch/riscv/include/asm/jump_label.h +++ b/arch/riscv/include/asm/jump_label.h @@ -16,21 +16,28 @@ #define JUMP_LABEL_NOP_SIZE 4 +#define JUMP_TABLE_ENTRY(key, label) \ + ".pushsection __jump_table, \"aw\" \n\t" \ + ".align " RISCV_LGPTR " \n\t" \ + ".long 1b - ., " label " - . \n\t" \ + "" RISCV_PTR " " key " - . \n\t" \ + ".popsection \n\t" + +/* This macro is also expanded on the Rust side. */ +#define ARCH_STATIC_BRANCH_ASM(key, label) \ + " .align 2 \n\t" \ + " .option push \n\t" \ + " .option norelax \n\t" \ + " .option norvc \n\t" \ + "1: j " label " \n\t" \ + " .option pop \n\t" \ + JUMP_TABLE_ENTRY(key, label) + static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch) { asm goto( - " .align 2 \n\t" - " .option push \n\t" - " .option norelax \n\t" - " .option norvc \n\t" - "1: nop \n\t" - " .option pop \n\t" - " .pushsection __jump_table, \"aw\" \n\t" - " .align " RISCV_LGPTR " \n\t" - " .long 1b - ., %l[label] - . \n\t" - " " RISCV_PTR " %0 - . \n\t" - " .popsection \n\t" + ARCH_STATIC_BRANCH_ASM("%0", "%l[label]") : : "i"(&((char *)key)[branch]) : : label); return false; @@ -38,21 +45,20 @@ static __always_inline bool arch_static_branch(struct static_key * const key, return true; } +#define ARCH_STATIC_BRANCH_JUMP_ASM(key, label) \ + " .align 2 \n\t" \ + " .option push \n\t" \ + " .option norelax \n\t" \ + " .option norvc \n\t" \ + "1: nop \n\t" \ + " .option pop \n\t" \ + JUMP_TABLE_ENTRY(key, label) + static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch) { asm goto( - " .align 2 \n\t" - " .option push \n\t" - " .option norelax \n\t" - " .option norvc \n\t" - "1: j %l[label] \n\t" - " .option pop \n\t" - " .pushsection __jump_table, \"aw\" \n\t" - " .align " RISCV_LGPTR " \n\t" - " .long 1b - ., %l[label] - . \n\t" - " " RISCV_PTR " %0 - . \n\t" - " .popsection \n\t" + ARCH_STATIC_BRANCH_JUMP_ASM("%0", "%l[label]") : : "i"(&((char *)key)[branch]) : : label); return false; diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h index cbbef32517f0..87a3a0f5bd22 100644 --- a/arch/x86/include/asm/jump_label.h +++ b/arch/x86/include/asm/jump_label.h @@ -12,49 +12,41 @@ #include <linux/stringify.h> #include <linux/types.h> -#define JUMP_TABLE_ENTRY \ +#define JUMP_TABLE_ENTRY(key, label) \ ".pushsection __jump_table, \"aw\" \n\t" \ _ASM_ALIGN "\n\t" \ ".long 1b - . \n\t" \ - ".long %l[l_yes] - . \n\t" \ - _ASM_PTR "%c0 + %c1 - .\n\t" \ + ".long " label "- . \n\t" \ + _ASM_PTR " " key " - . \n\t" \ ".popsection \n\t" +/* This macro is also expanded on the Rust side. */ #ifdef CONFIG_HAVE_JUMP_LABEL_HACK - -static __always_inline bool arch_static_branch(struct static_key *key, bool branch) -{ - asm goto("1:" - "jmp %l[l_yes] # objtool NOPs this \n\t" - JUMP_TABLE_ENTRY - : : "i" (key), "i" (2 | branch) : : l_yes); - - return false; -l_yes: - return true; -} - +#define ARCH_STATIC_BRANCH_ASM(key, label) \ + "1: jmp " label " # objtool NOPs this \n\t" \ + JUMP_TABLE_ENTRY(key, label) #else /* !CONFIG_HAVE_JUMP_LABEL_HACK */ +#define ARCH_STATIC_BRANCH_ASM(key, label) \ + "1: .byte " __stringify(BYTES_NOP5) "\n\t" \ + JUMP_TABLE_ENTRY(key, label) +#endif /* CONFIG_HAVE_JUMP_LABEL_HACK */ static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch) { - asm goto("1:" - ".byte " __stringify(BYTES_NOP5) "\n\t" - JUMP_TABLE_ENTRY - : : "i" (key), "i" (branch) : : l_yes); + int hack_bit = IS_ENABLED(CONFIG_HAVE_JUMP_LABEL_HACK) ? 2 : 0; + asm goto(ARCH_STATIC_BRANCH_ASM("%c0 + %c1", "%l[l_yes]") + : : "i" (key), "i" (hack_bit | branch) : : l_yes); return false; l_yes: return true; } -#endif /* CONFIG_HAVE_JUMP_LABEL_HACK */ - static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch) { asm goto("1:" "jmp %l[l_yes]\n\t" - JUMP_TABLE_ENTRY + JUMP_TABLE_ENTRY("%c0 + %c1", "%l[l_yes]") : : "i" (key), "i" (branch) : : l_yes); return false;
To avoid duplication of inline asm between C and Rust, we need to import the inline asm from the relevant `jump_label.h` header into Rust. To make that easier, this patch updates the header files to expose the inline asm via a new ARCH_STATIC_BRANCH_ASM macro. The header files are all updated to define a ARCH_STATIC_BRANCH_ASM that takes the same arguments in a consistent order so that Rust can use the same logic for every architecture. Link: https://lore.kernel.org/r/20240725183325.122827-7-ojeda@kernel.org [1] Signed-off-by: Alice Ryhl <aliceryhl@google.com> --- arch/arm/include/asm/jump_label.h | 14 +++++---- arch/arm64/include/asm/jump_label.h | 20 ++++++++----- arch/loongarch/include/asm/jump_label.h | 16 +++++++---- arch/riscv/include/asm/jump_label.h | 50 ++++++++++++++++++--------------- arch/x86/include/asm/jump_label.h | 38 ++++++++++--------------- 5 files changed, 75 insertions(+), 63 deletions(-)