From patchwork Mon Dec 4 09:37:03 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Peter Zijlstra X-Patchwork-Id: 13478006 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="VhetepAF" Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 2A305B2; Mon, 4 Dec 2023 01:39:57 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=3hjZXdVdAqYrfJGPL+xc1qQwDrkxAo6hzBpHpkHpUzk=; b=VhetepAFNhvW4k2HRO7V2nJ4vR yfQoimIyxY33PBF6OS0Gxx7UpZ6oYKJsHjx3CObRPboc8sMlMCepKgG9lqgqSLxCut+k0ZvQa0fc9 6c16CXtFn64ZgJzQ2HWq6vDQ7hc7W4ABXFJffLvRaLN+ekyOulZg4NTXqizEi0tbi6Su+cfJp9TNP 4jehR/v76SFY3vQtk2AbojhYeerO2qs7aXUZy/F4Ial8uKLGrAOzpYTIsvsUhZhyDhg/nbwzKvMUx GNInGk9bmIS6JI0CCwlJFBdouQ/XZ5ee/BTOCuhHJEerstoW9kLt+t7t/GCOUPIcsuf0li8qkLuBu qydLDGfQ==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1rA5QY-000X0N-GA; Mon, 04 Dec 2023 09:39:46 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 8BBD3300472; Mon, 4 Dec 2023 10:39:45 +0100 (CET) Message-Id: <20231204093731.356358182@infradead.org> User-Agent: quilt/0.65 Date: Mon, 04 Dec 2023 10:37:03 +0100 From: Peter Zijlstra To: Sean Christopherson , Paolo Bonzini , Josh Poimboeuf , Thomas Gleixner Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, x86@kernel.org, kvm@vger.kernel.org Subject: [PATCH 01/11] objtool: Generic annotation infrastructure References: <20231204093702.989848513@infradead.org> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Avoid endless .discard.foo sections for each annotation, create a single .discard.annotate section that takes an annotation type along with the instruction. Signed-off-by: Peter Zijlstra (Intel) --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -57,6 +57,13 @@ ".long 998b\n\t" \ ".popsection\n\t" +#define ASM_ANNOTATE(x) \ + "911:\n\t" \ + ".pushsection .discard.annotate,\"M\",@progbits,8\n\t" \ + ".long 911b - .\n\t" \ + ".long " __stringify(x) "\n\t" \ + ".popsection\n\t" + #else /* __ASSEMBLY__ */ /* @@ -146,6 +153,14 @@ .popsection .endm +.macro ANNOTATE type:req +.Lhere_\@: + .pushsection .discard.annotate,"M",@progbits,8 + .long .Lhere_\@ - . + .long \type + .popsection +.endm + #endif /* __ASSEMBLY__ */ #else /* !CONFIG_OBJTOOL */ @@ -167,6 +182,8 @@ .endm .macro REACHABLE .endm +.macro ANNOTATE +.endm #endif #endif /* CONFIG_OBJTOOL */ --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2308,6 +2308,41 @@ static int read_unwind_hints(struct objt return 0; } +static int read_annotate(struct objtool_file *file, void (*func)(int type, struct instruction *insn)) +{ + struct section *rsec, *sec; + struct instruction *insn; + struct reloc *reloc; + int type; + + rsec = find_section_by_name(file->elf, ".rela.discard.annotate"); + if (!rsec) + return 0; + + sec = find_section_by_name(file->elf, ".discard.annotate"); + if (!sec) + return 0; + + for_each_reloc(rsec, reloc) { + insn = find_insn(file, reloc->sym->sec, + reloc->sym->offset + reloc_addend(reloc)); + if (!insn) { + WARN("bad .discard.annotate entry: %d", reloc_idx(reloc)); + return -1; + } + + type = *(u32 *)(sec->data->d_buf + (reloc_idx(reloc) * sec->sh.sh_entsize) + 4); + + func(type, insn); + } + + return 0; +} + +static void __annotate_nop(int type, struct instruction *insn) +{ +} + static int read_noendbr_hints(struct objtool_file *file) { struct instruction *insn; @@ -2602,6 +2637,8 @@ static int decode_sections(struct objtoo if (ret) return ret; + ret = read_annotate(file, __annotate_nop); + /* * Must be before read_unwind_hints() since that needs insn->noendbr. */ From patchwork Mon Dec 4 09:37:04 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Peter Zijlstra X-Patchwork-Id: 13478003 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="UViUllN2" Received: from desiato.infradead.org (desiato.infradead.org [IPv6:2001:8b0:10b:1:d65d:64ff:fe57:4e05]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 2BA41D7; Mon, 4 Dec 2023 01:39:53 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=dYFXJA1/fZVGzxQ5AQj4cq8phZVmCHY7eA+a7xfnNgw=; b=UViUllN2GgMzBFM3XF7KPRY00g gP45Jx1xsIlrgGmfRzKuGONtQnc5C+nuQDeZtLxcuyeiXfyJa62wIwK4GwrlE3rd5IliR1jILRQ7F 69cFXzfDhlRmGLO8idhpw1eZGUftm4/uQkkiI54rs8GQvNVdGbb4cM4ehtU0rL1ZqlpoiSzIz2W/6 OhFYJ0bq4520m8JqEkRPTo5d8F6TLFuD05vKP2AcrvAMvzQRe7e1VFf8WVr689HlMsh+LXLQ8JkUS RJeSarJ9wt9MZ081Cz3YRcI0yBglq18QWBZUeaUi3kMwydt2ms3/y7ZcEeS0S2DGjXUMKRroIyYnN 5czykVNA==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.96 #2 (Red Hat Linux)) id 1rA5QX-004KSa-31; Mon, 04 Dec 2023 09:39:46 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 8FD413004AB; Mon, 4 Dec 2023 10:39:45 +0100 (CET) Message-Id: <20231204093731.465944066@infradead.org> User-Agent: quilt/0.65 Date: Mon, 04 Dec 2023 10:37:04 +0100 From: Peter Zijlstra To: Sean Christopherson , Paolo Bonzini , Josh Poimboeuf , Thomas Gleixner Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, x86@kernel.org, kvm@vger.kernel.org Subject: [PATCH 02/11] objtool: Convert ANNOTATE_NOENDBR to ANNOTATE References: <20231204093702.989848513@infradead.org> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Signed-off-by: Peter Zijlstra (Intel) --- include/linux/objtool.h | 17 ++++------------- include/linux/objtool_types.h | 5 +++++ tools/include/linux/objtool_types.h | 5 +++++ tools/objtool/check.c | 32 +++++--------------------------- 4 files changed, 19 insertions(+), 40 deletions(-) --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -45,12 +45,6 @@ #define STACK_FRAME_NON_STANDARD_FP(func) #endif -#define ANNOTATE_NOENDBR \ - "986: \n\t" \ - ".pushsection .discard.noendbr\n\t" \ - ".long 986b\n\t" \ - ".popsection\n\t" - #define ASM_REACHABLE \ "998:\n\t" \ ".pushsection .discard.reachable\n\t" \ @@ -64,6 +58,8 @@ ".long " __stringify(x) "\n\t" \ ".popsection\n\t" +#define ANNOTATE_NOENDBR ASM_ANNOTATE(ANNOTYPE_NOENDBR) + #else /* __ASSEMBLY__ */ /* @@ -122,13 +118,6 @@ #endif .endm -.macro ANNOTATE_NOENDBR -.Lhere_\@: - .pushsection .discard.noendbr - .long .Lhere_\@ - .popsection -.endm - /* * Use objtool to validate the entry requirement that all code paths do * VALIDATE_UNRET_END before RET. @@ -161,6 +150,8 @@ .popsection .endm +#define ANNOTATE_NOENDBR ANNOTATE type=ANNOTYPE_NOENDBR + #endif /* __ASSEMBLY__ */ #else /* !CONFIG_OBJTOOL */ --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -54,4 +54,9 @@ struct unwind_hint { #define UNWIND_HINT_TYPE_SAVE 6 #define UNWIND_HINT_TYPE_RESTORE 7 +/* + * Annotate types + */ +#define ANNOTYPE_NOENDBR 1 + #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -54,4 +54,9 @@ struct unwind_hint { #define UNWIND_HINT_TYPE_SAVE 6 #define UNWIND_HINT_TYPE_RESTORE 7 +/* + * Annotate types + */ +#define ANNOTYPE_NOENDBR 1 + #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2339,32 +2339,12 @@ static int read_annotate(struct objtool_ return 0; } -static void __annotate_nop(int type, struct instruction *insn) +static void __annotate_noendbr(int type, struct instruction *insn) { -} - -static int read_noendbr_hints(struct objtool_file *file) -{ - struct instruction *insn; - struct section *rsec; - struct reloc *reloc; - - rsec = find_section_by_name(file->elf, ".rela.discard.noendbr"); - if (!rsec) - return 0; + if (type != ANNOTYPE_NOENDBR) + return; - for_each_reloc(rsec, reloc) { - insn = find_insn(file, reloc->sym->sec, - reloc->sym->offset + reloc_addend(reloc)); - if (!insn) { - WARN("bad .discard.noendbr entry"); - return -1; - } - - insn->noendbr = 1; - } - - return 0; + insn->noendbr = 1; } static int read_retpoline_hints(struct objtool_file *file) @@ -2637,12 +2617,10 @@ static int decode_sections(struct objtoo if (ret) return ret; - ret = read_annotate(file, __annotate_nop); - /* * Must be before read_unwind_hints() since that needs insn->noendbr. */ - ret = read_noendbr_hints(file); + ret = read_annotate(file, __annotate_noendbr); if (ret) return ret; From patchwork Mon Dec 4 09:37:05 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Peter Zijlstra X-Patchwork-Id: 13478007 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="q9R9fx4v" Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id E2186F5; Mon, 4 Dec 2023 01:39:56 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=O8rCIvUkqNpAaMxQiAO/KoN2MOG0w6rqWm9LpswbbCQ=; b=q9R9fx4vg0uquJMFeJdbApPPIs jj3YYjiouv/ehAon4kJDHUvxLjVTIC8OrzcWy7fiomAUv1Er/dcezIlQ8DeuS+Y1CyG65cxsxhDwV 5WDIS9FLy8TwVzDjTXXwn6UYaK18UXcT4+9r+MoR1DKzKY9brq3tZnVXEPVyRcnSZNFdarWKphrlY K+kCOQKhbOc4WnO9ZZUCTAuMopdcSwFn9zma0LOdwI4Ug8F+PFzJhSGaQAWOM+gzQLZRsxSNVeW63 Wa2FwhIuOAUpsfKoAkpRf1h2gd49TvpZrQIe6CWpIVrabOJc2xh+IeHkzEJTUK/sdCt3YK5Au34aW 07PArYAg==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1rA5QY-000X0O-GF; Mon, 04 Dec 2023 09:39:46 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 9430E3004D5; Mon, 4 Dec 2023 10:39:45 +0100 (CET) Message-Id: <20231204093731.574465649@infradead.org> User-Agent: quilt/0.65 Date: Mon, 04 Dec 2023 10:37:05 +0100 From: Peter Zijlstra To: Sean Christopherson , Paolo Bonzini , Josh Poimboeuf , Thomas Gleixner Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, x86@kernel.org, kvm@vger.kernel.org Subject: [PATCH 03/11] objtool: Convert ANNOTATE_RETPOLINE_SAFE to ANNOTATE References: <20231204093702.989848513@infradead.org> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Signed-off-by: Peter Zijlstra (Intel) --- arch/x86/include/asm/nospec-branch.h | 13 +------- include/linux/objtool_types.h | 1 tools/include/linux/objtool_types.h | 1 tools/objtool/check.c | 52 ++++++++++++----------------------- 4 files changed, 22 insertions(+), 45 deletions(-) --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -193,12 +193,7 @@ * objtool the subsequent indirect jump/call is vouched safe for retpoline * builds. */ -.macro ANNOTATE_RETPOLINE_SAFE -.Lhere_\@: - .pushsection .discard.retpoline_safe - .long .Lhere_\@ - .popsection -.endm +#define ANNOTATE_RETPOLINE_SAFE ANNOTATE type=ANNOTYPE_RETPOLINE_SAFE /* * (ab)use RETPOLINE_SAFE on RET to annotate away 'bare' RET instructions @@ -317,11 +312,7 @@ #else /* __ASSEMBLY__ */ -#define ANNOTATE_RETPOLINE_SAFE \ - "999:\n\t" \ - ".pushsection .discard.retpoline_safe\n\t" \ - ".long 999b\n\t" \ - ".popsection\n\t" +#define ANNOTATE_RETPOLINE_SAFE ASM_ANNOTATE(ANNOTYPE_RETPOLINE_SAFE) typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE]; extern retpoline_thunk_t __x86_indirect_thunk_array[]; --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -58,5 +58,6 @@ struct unwind_hint { * Annotate types */ #define ANNOTYPE_NOENDBR 1 +#define ANNOTYPE_RETPOLINE_SAFE 2 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -58,5 +58,6 @@ struct unwind_hint { * Annotate types */ #define ANNOTYPE_NOENDBR 1 +#define ANNOTYPE_RETPOLINE_SAFE 2 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2308,12 +2308,12 @@ static int read_unwind_hints(struct objt return 0; } -static int read_annotate(struct objtool_file *file, void (*func)(int type, struct instruction *insn)) +static int read_annotate(struct objtool_file *file, int (*func)(int type, struct instruction *insn)) { struct section *rsec, *sec; struct instruction *insn; struct reloc *reloc; - int type; + int type, ret; rsec = find_section_by_name(file->elf, ".rela.discard.annotate"); if (!rsec) @@ -2333,53 +2333,37 @@ static int read_annotate(struct objtool_ type = *(u32 *)(sec->data->d_buf + (reloc_idx(reloc) * sec->sh.sh_entsize) + 4); - func(type, insn); + ret = func(type, insn); + if (ret < 0) + return ret; } return 0; } -static void __annotate_noendbr(int type, struct instruction *insn) +static int __annotate_noendbr(int type, struct instruction *insn) { if (type != ANNOTYPE_NOENDBR) - return; + return 0; insn->noendbr = 1; + return 0; } -static int read_retpoline_hints(struct objtool_file *file) +static int __annotate_retpoline_safe(int type, struct instruction *insn) { - struct section *rsec; - struct instruction *insn; - struct reloc *reloc; - - rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe"); - if (!rsec) + if (type != ANNOTYPE_RETPOLINE_SAFE) return 0; - for_each_reloc(rsec, reloc) { - if (reloc->sym->type != STT_SECTION) { - WARN("unexpected relocation symbol type in %s", rsec->name); - return -1; - } - - insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); - if (!insn) { - WARN("bad .discard.retpoline_safe entry"); - return -1; - } - - if (insn->type != INSN_JUMP_DYNAMIC && - insn->type != INSN_CALL_DYNAMIC && - insn->type != INSN_RETURN && - insn->type != INSN_NOP) { - WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop"); - return -1; - } - - insn->retpoline_safe = true; + if (insn->type != INSN_JUMP_DYNAMIC && + insn->type != INSN_CALL_DYNAMIC && + insn->type != INSN_RETURN && + insn->type != INSN_NOP) { + WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop"); + return -1; } + insn->retpoline_safe = true; return 0; } @@ -2666,7 +2650,7 @@ static int decode_sections(struct objtoo if (ret) return ret; - ret = read_retpoline_hints(file); + ret = read_annotate(file, __annotate_retpoline_safe); if (ret) return ret; From patchwork Mon Dec 4 09:37:06 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Peter Zijlstra X-Patchwork-Id: 13478010 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="LSDJhy0t" Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 561FA106; Mon, 4 Dec 2023 01:39:58 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=gFiDthjTb2jV5OQYX2IoNEzMxtX8OE63urpzPswRdWY=; b=LSDJhy0tFFO/L0bq9COPVzaqVF bhDhZ3JnjeXxphLgY4flbljhmu/U5KIhZ6A/V5+ElaDcouYjY7Gy845lVJaZAU4tz0Xyi8/cyFyPz SMKABRRODkBBnKKrE2tSIrjil13QBM+oG84AEqld5hzCLDc9swSnXMRdAv67oS9MB+TZj4r0qAHz4 owfWdwA96RFUMdtT77j5fIG+5WyL/BfpTYbzwxa0iOes2pqQQmnVLt7nyhI9F+5mWAu5tXn1Qi/Fj gCKL29MU4a/jhxpHMLtJUmBEYdl/hdWulzGAiwOJjNRhWEenD7I/zxkQolUjeSOkt+TixPBWcuBul kRyGmS5A==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1rA5QY-000X0P-GC; Mon, 04 Dec 2023 09:39:46 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 97E073005B2; Mon, 4 Dec 2023 10:39:45 +0100 (CET) Message-Id: <20231204093731.682850335@infradead.org> User-Agent: quilt/0.65 Date: Mon, 04 Dec 2023 10:37:06 +0100 From: Peter Zijlstra To: Sean Christopherson , Paolo Bonzini , Josh Poimboeuf , Thomas Gleixner Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, x86@kernel.org, kvm@vger.kernel.org Subject: [PATCH 04/11] objtool: Convert instrumentation_{begin,end}() to ANNOTATE References: <20231204093702.989848513@infradead.org> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Signed-off-by: Peter Zijlstra (Intel) --- include/linux/instrumentation.h | 11 +++----- include/linux/objtool.h | 9 ++++-- include/linux/objtool_types.h | 2 + tools/include/linux/objtool_types.h | 2 + tools/objtool/check.c | 49 +++++++----------------------------- 5 files changed, 25 insertions(+), 48 deletions(-) --- a/include/linux/instrumentation.h +++ b/include/linux/instrumentation.h @@ -4,14 +4,14 @@ #ifdef CONFIG_NOINSTR_VALIDATION +#include #include /* Begin/end of an instrumentation safe region */ #define __instrumentation_begin(c) ({ \ asm volatile(__stringify(c) ": nop\n\t" \ - ".pushsection .discard.instr_begin\n\t" \ - ".long " __stringify(c) "b - .\n\t" \ - ".popsection\n\t" : : "i" (c)); \ + __ASM_ANNOTATE(c, ANNOTYPE_INSTR_BEGIN) \ + : : "i" (c)); \ }) #define instrumentation_begin() __instrumentation_begin(__COUNTER__) @@ -48,9 +48,8 @@ */ #define __instrumentation_end(c) ({ \ asm volatile(__stringify(c) ": nop\n\t" \ - ".pushsection .discard.instr_end\n\t" \ - ".long " __stringify(c) "b - .\n\t" \ - ".popsection\n\t" : : "i" (c)); \ + __ASM_ANNOTATE(c, ANNOTYPE_INSTR_END) \ + : : "i" (c)); \ }) #define instrumentation_end() __instrumentation_end(__COUNTER__) #else /* !CONFIG_NOINSTR_VALIDATION */ --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -51,13 +51,16 @@ ".long 998b\n\t" \ ".popsection\n\t" -#define ASM_ANNOTATE(x) \ - "911:\n\t" \ +#define __ASM_ANNOTATE(s, x) \ ".pushsection .discard.annotate,\"M\",@progbits,8\n\t" \ - ".long 911b - .\n\t" \ + ".long " __stringify(s) "b - .\n\t" \ ".long " __stringify(x) "\n\t" \ ".popsection\n\t" +#define ASM_ANNOTATE(x) \ + "911:\n\t" \ + __ASM_ANNOTATE(911, x) + #define ANNOTATE_NOENDBR ASM_ANNOTATE(ANNOTYPE_NOENDBR) #else /* __ASSEMBLY__ */ --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -59,5 +59,7 @@ struct unwind_hint { */ #define ANNOTYPE_NOENDBR 1 #define ANNOTYPE_RETPOLINE_SAFE 2 +#define ANNOTYPE_INSTR_BEGIN 3 +#define ANNOTYPE_INSTR_END 4 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -59,5 +59,7 @@ struct unwind_hint { */ #define ANNOTYPE_NOENDBR 1 #define ANNOTYPE_RETPOLINE_SAFE 2 +#define ANNOTYPE_INSTR_BEGIN 3 +#define ANNOTYPE_INSTR_END 4 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2367,48 +2367,19 @@ static int __annotate_retpoline_safe(int return 0; } -static int read_instr_hints(struct objtool_file *file) +static int __annotate_instr(int type, struct instruction *insn) { - struct section *rsec; - struct instruction *insn; - struct reloc *reloc; - - rsec = find_section_by_name(file->elf, ".rela.discard.instr_end"); - if (!rsec) - return 0; - - for_each_reloc(rsec, reloc) { - if (reloc->sym->type != STT_SECTION) { - WARN("unexpected relocation symbol type in %s", rsec->name); - return -1; - } - - insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); - if (!insn) { - WARN("bad .discard.instr_end entry"); - return -1; - } + switch (type) { + case ANNOTYPE_INSTR_BEGIN: + insn->instr++; + break; + case ANNOTYPE_INSTR_END: insn->instr--; - } - - rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin"); - if (!rsec) - return 0; + break; - for_each_reloc(rsec, reloc) { - if (reloc->sym->type != STT_SECTION) { - WARN("unexpected relocation symbol type in %s", rsec->name); - return -1; - } - - insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); - if (!insn) { - WARN("bad .discard.instr_begin entry"); - return -1; - } - - insn->instr++; + default: + break; } return 0; @@ -2654,7 +2625,7 @@ static int decode_sections(struct objtoo if (ret) return ret; - ret = read_instr_hints(file); + ret = read_annotate(file, __annotate_instr); if (ret) return ret; From patchwork Mon Dec 4 09:37:07 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Peter Zijlstra X-Patchwork-Id: 13478001 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="Rgb1sTCn" Received: from desiato.infradead.org (desiato.infradead.org [IPv6:2001:8b0:10b:1:d65d:64ff:fe57:4e05]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 60B86D2; Mon, 4 Dec 2023 01:39:52 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=g0iuyQNisDJC7BhdoXr7Rs7MdGYNzIIaLt95P6Iehys=; b=Rgb1sTCnRXLp1JXvxpNMSDS/1Z mIlwpaJ5yxcl7ADLIOhucqUrAEp0vm0h502BryZed9fq+raHpUTD06iVrwRkDVKZUj40MSRckEJgU 7MIqcb0uwH63f0WuPRUSVFkK4NNHQCKRp1GMY3nqKMgEPxuP3NWjf1ma/XmwpA1Rxe4sApMJIlTF7 pG7TeZv7icS38Q7qGYlXNqrJ04SJZZoqdiuc5rqRgl8BYUagwDU7t9cCcqgfDIG22izfvK+0sMBqq rd51IgIfKKvvRyhd56KWi+xXipG1WtvGZ/KLyVCjEjXzWjz8TgFv1AFtqWboNrrI1RkPb+DVuUEhY GEDJLQlA==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.96 #2 (Red Hat Linux)) id 1rA5QZ-004KSf-0G; Mon, 04 Dec 2023 09:39:47 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 9C995300665; Mon, 4 Dec 2023 10:39:45 +0100 (CET) Message-Id: <20231204093731.812275775@infradead.org> User-Agent: quilt/0.65 Date: Mon, 04 Dec 2023 10:37:07 +0100 From: Peter Zijlstra To: Sean Christopherson , Paolo Bonzini , Josh Poimboeuf , Thomas Gleixner Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, x86@kernel.org, kvm@vger.kernel.org Subject: [PATCH 05/11] objtool: Convert VALIDATE_UNRET_BEGIN to ANNOTATE References: <20231204093702.989848513@infradead.org> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Signed-off-by: Peter Zijlstra (Intel) --- include/linux/objtool.h | 9 +++------ include/linux/objtool_types.h | 1 + tools/include/linux/objtool_types.h | 1 + tools/objtool/check.c | 28 +++++----------------------- 4 files changed, 10 insertions(+), 29 deletions(-) --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -128,15 +128,12 @@ * NOTE: The macro must be used at the beginning of a global symbol, otherwise * it will be ignored. */ -.macro VALIDATE_UNRET_BEGIN #if defined(CONFIG_NOINSTR_VALIDATION) && \ (defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_SRSO)) -.Lhere_\@: - .pushsection .discard.validate_unret - .long .Lhere_\@ - . - .popsection +#define VALIDATE_UNRET_BEGIN ANNOTATE type=ANNOTYPE_UNRET_BEGIN +#else +#define VALIDATE_UNRET_BEGIN #endif -.endm .macro REACHABLE .Lhere_\@: --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -61,5 +61,6 @@ struct unwind_hint { #define ANNOTYPE_RETPOLINE_SAFE 2 #define ANNOTYPE_INSTR_BEGIN 3 #define ANNOTYPE_INSTR_END 4 +#define ANNOTYPE_UNRET_BEGIN 5 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -61,5 +61,6 @@ struct unwind_hint { #define ANNOTYPE_RETPOLINE_SAFE 2 #define ANNOTYPE_INSTR_BEGIN 3 #define ANNOTYPE_INSTR_END 4 +#define ANNOTYPE_UNRET_BEGIN 5 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2385,33 +2385,15 @@ static int __annotate_instr(int type, st return 0; } -static int read_validate_unret_hints(struct objtool_file *file) +static int __annotate_unret(int type, struct instruction *insn) { - struct section *rsec; - struct instruction *insn; - struct reloc *reloc; - - rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret"); - if (!rsec) + if (type != ANNOTYPE_UNRET_BEGIN) return 0; - for_each_reloc(rsec, reloc) { - if (reloc->sym->type != STT_SECTION) { - WARN("unexpected relocation symbol type in %s", rsec->name); - return -1; - } - - insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); - if (!insn) { - WARN("bad .discard.instr_end entry"); - return -1; - } - insn->unret = 1; - } - + insn->unret = 1; return 0; -} +} static int read_intra_function_calls(struct objtool_file *file) { @@ -2629,7 +2611,7 @@ static int decode_sections(struct objtoo if (ret) return ret; - ret = read_validate_unret_hints(file); + ret = read_annotate(file, __annotate_unret); if (ret) return ret; From patchwork Mon Dec 4 09:37:08 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Peter Zijlstra X-Patchwork-Id: 13478002 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="hp565mu6" Received: from desiato.infradead.org (desiato.infradead.org [IPv6:2001:8b0:10b:1:d65d:64ff:fe57:4e05]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 1DAF0B2; Mon, 4 Dec 2023 01:39:52 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=iT8rZBU1kFBBncA0eT4LO89fZPLO0VJsqubseEy0Iso=; b=hp565mu6qAqH0rQKKpsy33mH+O VpKF9v1+ZY5Yp+5PFrwbFzKoNR9q9e3UR44zfU8E52uaGpF57V1ZBwk9ex1c/RW5p3Od/YJFqJZz5 zuiTDQVT9+w7JKjVWBcLB8Vv4+gmq129P9lBr5yg9qvgqBxB4BrFavooC1bGcArbIfucbEELn6o79 fR1WgLHahQIucmH/dpwTyvpF1k5sfKeNNizLgH/ADqofQuJ1dUJ8NWxTTlHcTWnw15v19iGIQ5nGR Kv+xQU8tCFXSNvMjw6+2brg+Q6jtAnzfPE6Nd+wgZr7wajPnlEpn58wxBW181sjQQUqWc1YGByOod yXtVUwng==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.96 #2 (Red Hat Linux)) id 1rA5QY-004KSg-34; Mon, 04 Dec 2023 09:39:47 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id A03D730088E; Mon, 4 Dec 2023 10:39:45 +0100 (CET) Message-Id: <20231204093731.986118946@infradead.org> User-Agent: quilt/0.65 Date: Mon, 04 Dec 2023 10:37:08 +0100 From: Peter Zijlstra To: Sean Christopherson , Paolo Bonzini , Josh Poimboeuf , Thomas Gleixner Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, x86@kernel.org, kvm@vger.kernel.org Subject: [PATCH 06/11] objtool: Convert ANNOTATE_IGNORE_ALTERNATIVE to ANNOTATE References: <20231204093702.989848513@infradead.org> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Signed-off-by: Peter Zijlstra (Intel) --- arch/x86/include/asm/alternative.h | 14 ++--------- include/linux/objtool_types.h | 1 tools/include/linux/objtool_types.h | 1 tools/objtool/check.c | 45 ++++++++---------------------------- 4 files changed, 15 insertions(+), 46 deletions(-) --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -4,6 +4,7 @@ #include #include +#include #include #define ALT_FLAGS_SHIFT 16 @@ -55,11 +56,7 @@ * objtool annotation to ignore the alternatives and only consider the original * instruction(s). */ -#define ANNOTATE_IGNORE_ALTERNATIVE \ - "999:\n\t" \ - ".pushsection .discard.ignore_alts\n\t" \ - ".long 999b\n\t" \ - ".popsection\n\t" +#define ANNOTATE_IGNORE_ALTERNATIVE ASM_ANNOTATE(ANNOTYPE_IGNORE_ALTS) /* * The patching flags are part of the upper bits of the @ft_flags parameter when @@ -349,12 +346,7 @@ static inline int alternatives_text_rese * objtool annotation to ignore the alternatives and only consider the original * instruction(s). */ -.macro ANNOTATE_IGNORE_ALTERNATIVE - .Lannotate_\@: - .pushsection .discard.ignore_alts - .long .Lannotate_\@ - .popsection -.endm +#define ANNOTATE_IGNORE_ALTERNATIVE ANNOTATE type=ANNOTYPE_IGNORE_ALTS /* * Issue one struct alt_instr descriptor entry (need to put it into --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -62,5 +62,6 @@ struct unwind_hint { #define ANNOTYPE_INSTR_BEGIN 3 #define ANNOTYPE_INSTR_END 4 #define ANNOTYPE_UNRET_BEGIN 5 +#define ANNOTYPE_IGNORE_ALTS 6 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -62,5 +62,6 @@ struct unwind_hint { #define ANNOTYPE_INSTR_BEGIN 3 #define ANNOTYPE_INSTR_END 4 #define ANNOTYPE_UNRET_BEGIN 5 +#define ANNOTYPE_IGNORE_ALTS 6 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -1255,40 +1255,6 @@ static void add_uaccess_safe(struct objt } /* - * FIXME: For now, just ignore any alternatives which add retpolines. This is - * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline. - * But it at least allows objtool to understand the control flow *around* the - * retpoline. - */ -static int add_ignore_alternatives(struct objtool_file *file) -{ - struct section *rsec; - struct reloc *reloc; - struct instruction *insn; - - rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts"); - if (!rsec) - return 0; - - for_each_reloc(rsec, reloc) { - if (reloc->sym->type != STT_SECTION) { - WARN("unexpected relocation symbol type in %s", rsec->name); - return -1; - } - - insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); - if (!insn) { - WARN("bad .discard.ignore_alts entry"); - return -1; - } - - insn->ignore_alts = true; - } - - return 0; -} - -/* * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol * will be added to the .retpoline_sites section. */ @@ -2341,6 +2307,15 @@ static int read_annotate(struct objtool_ return 0; } +static int __annotate_ignore_alts(int type, struct instruction *insn) +{ + if (type != ANNOTYPE_IGNORE_ALTS) + return 0; + + insn->ignore_alts = true; + return 0; +} + static int __annotate_noendbr(int type, struct instruction *insn) { if (type != ANNOTYPE_NOENDBR) @@ -2550,7 +2525,7 @@ static int decode_sections(struct objtoo add_ignores(file); add_uaccess_safe(file); - ret = add_ignore_alternatives(file); + ret = read_annotate(file, __annotate_ignore_alts); if (ret) return ret; From patchwork Mon Dec 4 09:37:09 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Peter Zijlstra X-Patchwork-Id: 13478012 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="TsMsM4Rq" Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id DB1F1119; Mon, 4 Dec 2023 01:39:58 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=Jgweoy9v1ry+vJ+0xQvkdC/DCJR/HfQjV7GgwvonHBA=; b=TsMsM4RqmJ30Dxhaj7X6J4Zvnb S1vBqgDpLm/YetggJX/GKdtsZdMYIARCMQTuL89qct50tzNboez/lYKD5uO3NX2mOvnewAo+12F+P tZpcUqtnC7SQLklE7qJe72Y8wk4/wE6R/PWxK4aqk2QFRyxHDiqe4E773IZn/gqrE8ORLRFNzEPj6 LkyyKx7xzYzviXyYUkY6Xqo7NKc5tMTdkXO77Kgg2wlwR7I1pCrTfGT4ipektuCH7g26epOtf7N3y nNMKzkx8qfvtDKIsTMjFgAz4cUYY0bLyefv++/U3k+9SjN3c/ua7Nrll5njDV0bXDXarSA0nB3JGR R0S+ckng==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1rA5QZ-000X0Z-5i; Mon, 04 Dec 2023 09:39:47 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id A486B300AA5; Mon, 4 Dec 2023 10:39:45 +0100 (CET) Message-Id: <20231204093732.097671022@infradead.org> User-Agent: quilt/0.65 Date: Mon, 04 Dec 2023 10:37:09 +0100 From: Peter Zijlstra To: Sean Christopherson , Paolo Bonzini , Josh Poimboeuf , Thomas Gleixner Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, x86@kernel.org, kvm@vger.kernel.org Subject: [PATCH 07/11] objtool: Convert ANNOTATE_INTRA_FUNCTION_CALLS to ANNOTATE References: <20231204093702.989848513@infradead.org> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Signed-off-by: Peter Zijlstra (Intel) --- include/linux/objtool.h | 16 ++---- include/linux/objtool_types.h | 1 tools/include/linux/objtool_types.h | 1 tools/objtool/check.c | 96 ++++++++++++++---------------------- 4 files changed, 47 insertions(+), 67 deletions(-) --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -66,16 +66,6 @@ #else /* __ASSEMBLY__ */ /* - * This macro indicates that the following intra-function call is valid. - * Any non-annotated intra-function call will cause objtool to issue a warning. - */ -#define ANNOTATE_INTRA_FUNCTION_CALL \ - 999: \ - .pushsection .discard.intra_function_calls; \ - .long 999b; \ - .popsection; - -/* * In asm, there are two kinds of code: normal C-type callable functions and * the rest. The normal callable functions can be called by other code, and * don't do anything unusual with the stack. Such normal callable functions @@ -152,6 +142,12 @@ #define ANNOTATE_NOENDBR ANNOTATE type=ANNOTYPE_NOENDBR +/* + * This macro indicates that the following intra-function call is valid. + * Any non-annotated intra-function call will cause objtool to issue a warning. + */ +#define ANNOTATE_INTRA_FUNCTION_CALL ANNOTATE type=ANNOTYPE_INTRA_FUNCTION_CALLS + #endif /* __ASSEMBLY__ */ #else /* !CONFIG_OBJTOOL */ --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -63,5 +63,6 @@ struct unwind_hint { #define ANNOTYPE_INSTR_END 4 #define ANNOTYPE_UNRET_BEGIN 5 #define ANNOTYPE_IGNORE_ALTS 6 +#define ANNOTYPE_INTRA_FUNCTION_CALLS 7 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -63,5 +63,6 @@ struct unwind_hint { #define ANNOTYPE_INSTR_END 4 #define ANNOTYPE_UNRET_BEGIN 5 #define ANNOTYPE_IGNORE_ALTS 6 +#define ANNOTYPE_INTRA_FUNCTION_CALLS 7 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2274,7 +2274,8 @@ static int read_unwind_hints(struct objt return 0; } -static int read_annotate(struct objtool_file *file, int (*func)(int type, struct instruction *insn)) +static int read_annotate(struct objtool_file *file, + int (*func)(struct objtool_file *file, int type, struct instruction *insn)) { struct section *rsec, *sec; struct instruction *insn; @@ -2299,7 +2300,7 @@ static int read_annotate(struct objtool_ type = *(u32 *)(sec->data->d_buf + (reloc_idx(reloc) * sec->sh.sh_entsize) + 4); - ret = func(type, insn); + ret = func(file, type, insn); if (ret < 0) return ret; } @@ -2307,7 +2308,7 @@ static int read_annotate(struct objtool_ return 0; } -static int __annotate_ignore_alts(int type, struct instruction *insn) +static int __annotate_ignore_alts(struct objtool_file *file, int type, struct instruction *insn) { if (type != ANNOTYPE_IGNORE_ALTS) return 0; @@ -2316,7 +2317,7 @@ static int __annotate_ignore_alts(int ty return 0; } -static int __annotate_noendbr(int type, struct instruction *insn) +static int __annotate_noendbr(struct objtool_file *file, int type, struct instruction *insn) { if (type != ANNOTYPE_NOENDBR) return 0; @@ -2325,7 +2326,37 @@ static int __annotate_noendbr(int type, return 0; } -static int __annotate_retpoline_safe(int type, struct instruction *insn) +static int __annotate_ifc(struct objtool_file *file, int type, struct instruction *insn) +{ + unsigned long dest_off; + + if (type != ANNOTYPE_INTRA_FUNCTION_CALLS) + return 0; + + if (insn->type != INSN_CALL) { + WARN_INSN(insn, "intra_function_call not a direct call"); + return -1; + } + + /* + * Treat intra-function CALLs as JMPs, but with a stack_op. + * See add_call_destinations(), which strips stack_ops from + * normal CALLs. + */ + insn->type = INSN_JUMP_UNCONDITIONAL; + + dest_off = arch_jump_destination(insn); + insn->jump_dest = find_insn(file, insn->sec, dest_off); + if (!insn->jump_dest) { + WARN_INSN(insn, "can't find call dest at %s+0x%lx", + insn->sec->name, dest_off); + return -1; + } + + return 0; +} + +static int __annotate_retpoline_safe(struct objtool_file *file, int type, struct instruction *insn) { if (type != ANNOTYPE_RETPOLINE_SAFE) return 0; @@ -2342,7 +2373,7 @@ static int __annotate_retpoline_safe(int return 0; } -static int __annotate_instr(int type, struct instruction *insn) +static int __annotate_instr(struct objtool_file *file, int type, struct instruction *insn) { switch (type) { case ANNOTYPE_INSTR_BEGIN: @@ -2360,7 +2391,7 @@ static int __annotate_instr(int type, st return 0; } -static int __annotate_unret(int type, struct instruction *insn) +static int __annotate_unret(struct objtool_file *file, int type, struct instruction *insn) { if (type != ANNOTYPE_UNRET_BEGIN) return 0; @@ -2370,55 +2401,6 @@ static int __annotate_unret(int type, st } -static int read_intra_function_calls(struct objtool_file *file) -{ - struct instruction *insn; - struct section *rsec; - struct reloc *reloc; - - rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls"); - if (!rsec) - return 0; - - for_each_reloc(rsec, reloc) { - unsigned long dest_off; - - if (reloc->sym->type != STT_SECTION) { - WARN("unexpected relocation symbol type in %s", - rsec->name); - return -1; - } - - insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); - if (!insn) { - WARN("bad .discard.intra_function_call entry"); - return -1; - } - - if (insn->type != INSN_CALL) { - WARN_INSN(insn, "intra_function_call not a direct call"); - return -1; - } - - /* - * Treat intra-function CALLs as JMPs, but with a stack_op. - * See add_call_destinations(), which strips stack_ops from - * normal CALLs. - */ - insn->type = INSN_JUMP_UNCONDITIONAL; - - dest_off = arch_jump_destination(insn); - insn->jump_dest = find_insn(file, insn->sec, dest_off); - if (!insn->jump_dest) { - WARN_INSN(insn, "can't find call dest at %s+0x%lx", - insn->sec->name, dest_off); - return -1; - } - } - - return 0; -} - /* * Return true if name matches an instrumentation function, where calls to that * function from noinstr code can safely be removed, but compilers won't do so. @@ -2554,7 +2536,7 @@ static int decode_sections(struct objtoo * Must be before add_call_destination(); it changes INSN_CALL to * INSN_JUMP. */ - ret = read_intra_function_calls(file); + ret = read_annotate(file, __annotate_ifc); if (ret) return ret; From patchwork Mon Dec 4 09:37:10 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Peter Zijlstra X-Patchwork-Id: 13478011 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="FC/UR0Ci" Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 10D5511F; Mon, 4 Dec 2023 01:39:59 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=8ZmAUMpjWfI1B/Ma4euS1POKJrZM+zq/pwDtgORWvEc=; b=FC/UR0CiYDmD1lg2YNafE0u8li V7kroPRWHIgQ2/E/iiUueEIj1uUgoAEPxumPvomDI6eB1boaDNpS+xHF8wBxJZQpBWsZpXaJrm6Cc cK825dGuTUbyp02KfhCJtl+XVWRJws9bjmZLPmB1Z4y9uL4QVqNOm+XJaLk45wagD56VsPcIRzCXf zF3DadFti78h/5AlaHjW0ncY33DERXcbn0t3HXKvRF7BKgDDRUMUUKbQpiN7VZyL//20MiE6J8YHc nB22u8bGlf/mkrq8eV9wrtG3b4DseXA8uT1Kvzg7L+7BCOJ4P4H7KUoxUNV60xmQpTD43+sWYE8Qe YY8dUfaQ==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1rA5QZ-000X0a-66; Mon, 04 Dec 2023 09:39:47 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id A8328300FAE; Mon, 4 Dec 2023 10:39:45 +0100 (CET) Message-Id: <20231204093732.204792131@infradead.org> User-Agent: quilt/0.65 Date: Mon, 04 Dec 2023 10:37:10 +0100 From: Peter Zijlstra To: Sean Christopherson , Paolo Bonzini , Josh Poimboeuf , Thomas Gleixner Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, x86@kernel.org, kvm@vger.kernel.org Subject: [PATCH 08/11] objtool: Collapse annotate sequences References: <20231204093702.989848513@infradead.org> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Reduce read_annotate() runs by collapsing subsequent runs into a single call. Signed-off-by: Peter Zijlstra (Intel) --- tools/objtool/check.c | 87 ++++++++++++++++++-------------------------------- 1 file changed, 32 insertions(+), 55 deletions(-) --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2308,21 +2308,24 @@ static int read_annotate(struct objtool_ return 0; } -static int __annotate_ignore_alts(struct objtool_file *file, int type, struct instruction *insn) +static int __annotate_early(struct objtool_file *file, int type, struct instruction *insn) { - if (type != ANNOTYPE_IGNORE_ALTS) - return 0; + switch (type) { + case ANNOTYPE_IGNORE_ALTS: + insn->ignore_alts = true; + break; - insn->ignore_alts = true; - return 0; -} + /* + * Must be before read_unwind_hints() since that needs insn->noendbr. + */ + case ANNOTYPE_NOENDBR: + insn->noendbr = 1; + break; -static int __annotate_noendbr(struct objtool_file *file, int type, struct instruction *insn) -{ - if (type != ANNOTYPE_NOENDBR) - return 0; + default: + break; + } - insn->noendbr = 1; return 0; } @@ -2356,26 +2359,21 @@ static int __annotate_ifc(struct objtool return 0; } -static int __annotate_retpoline_safe(struct objtool_file *file, int type, struct instruction *insn) +static int __annotate_late(struct objtool_file *file, int type, struct instruction *insn) { - if (type != ANNOTYPE_RETPOLINE_SAFE) - return 0; - - if (insn->type != INSN_JUMP_DYNAMIC && - insn->type != INSN_CALL_DYNAMIC && - insn->type != INSN_RETURN && - insn->type != INSN_NOP) { - WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop"); - return -1; - } + switch (type) { + case ANNOTYPE_RETPOLINE_SAFE: + if (insn->type != INSN_JUMP_DYNAMIC && + insn->type != INSN_CALL_DYNAMIC && + insn->type != INSN_RETURN && + insn->type != INSN_NOP) { + WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop"); + return -1; + } - insn->retpoline_safe = true; - return 0; -} + insn->retpoline_safe = true; + break; -static int __annotate_instr(struct objtool_file *file, int type, struct instruction *insn) -{ - switch (type) { case ANNOTYPE_INSTR_BEGIN: insn->instr++; break; @@ -2384,6 +2382,10 @@ static int __annotate_instr(struct objto insn->instr--; break; + case ANNOTYPE_UNRET_BEGIN: + insn->unret = 1; + break; + default: break; } @@ -2391,16 +2393,6 @@ static int __annotate_instr(struct objto return 0; } -static int __annotate_unret(struct objtool_file *file, int type, struct instruction *insn) -{ - if (type != ANNOTYPE_UNRET_BEGIN) - return 0; - - insn->unret = 1; - return 0; - -} - /* * Return true if name matches an instrumentation function, where calls to that * function from noinstr code can safely be removed, but compilers won't do so. @@ -2507,14 +2499,7 @@ static int decode_sections(struct objtoo add_ignores(file); add_uaccess_safe(file); - ret = read_annotate(file, __annotate_ignore_alts); - if (ret) - return ret; - - /* - * Must be before read_unwind_hints() since that needs insn->noendbr. - */ - ret = read_annotate(file, __annotate_noendbr); + ret = read_annotate(file, __annotate_early); if (ret) return ret; @@ -2560,15 +2545,7 @@ static int decode_sections(struct objtoo if (ret) return ret; - ret = read_annotate(file, __annotate_retpoline_safe); - if (ret) - return ret; - - ret = read_annotate(file, __annotate_instr); - if (ret) - return ret; - - ret = read_annotate(file, __annotate_unret); + ret = read_annotate(file, __annotate_late); if (ret) return ret; From patchwork Mon Dec 4 09:37:11 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Peter Zijlstra X-Patchwork-Id: 13478009 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="HZt/sTRX" Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id E408D11A; Mon, 4 Dec 2023 01:39:58 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=hovFmglboAd1PTUKIc8q/eAFcPeS4SThi+Ejk6Ow87o=; b=HZt/sTRXVWWpY54+u0ELel8mQl r2VOv0Zmd6b7hGtm877EBapQsyIqKAEI7MecYEEXbV5Ci1995HCDmM71QGXQFpe68MDmbI8v6oMRQ C0C6R+H8/NModt5JmRNXzeYZNTMutMT55mv6lNr+H7a3EZU/O5HsQtQJWtgTV3FTXRTaMClSH/BAv bRj3WmOQHgh4kJ7ReCmzNKQrdl6XjcBLsmWaqXRWG6lUOGnaajRE7NneQR8vb8+WbN0dSItwfuG+R BaWzICX22woghbGQvQvFM1sHE6YD147/Na2aBwP4uHDzjoDzBG8zZevEoRjpAyFIxLwJuVjhpLD+4 OGzkgG2Q==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1rA5QZ-000X0b-6y; Mon, 04 Dec 2023 09:39:47 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id ABFC830198F; Mon, 4 Dec 2023 10:39:45 +0100 (CET) Message-Id: <20231204093732.323101886@infradead.org> User-Agent: quilt/0.65 Date: Mon, 04 Dec 2023 10:37:11 +0100 From: Peter Zijlstra To: Sean Christopherson , Paolo Bonzini , Josh Poimboeuf , Thomas Gleixner Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, x86@kernel.org, kvm@vger.kernel.org Subject: [PATCH 09/11] x86/kvm/emulate: Implement test_cc() in C References: <20231204093702.989848513@infradead.org> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Current test_cc() uses the fastop infrastructure to test flags using SETcc instructions. However, int3_emulate_jcc() already fully implements the flags->CC mapping, use that. Removes a pile of gnarly asm. Signed-off-by: Peter Zijlstra (Intel) --- arch/x86/include/asm/text-patching.h | 20 +++++++++++++------- arch/x86/kvm/emulate.c | 34 ++-------------------------------- 2 files changed, 15 insertions(+), 39 deletions(-) --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -186,9 +186,9 @@ void int3_emulate_ret(struct pt_regs *re } static __always_inline -void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp) +bool __emulate_cc(unsigned long flags, u8 cc) { - static const unsigned long jcc_mask[6] = { + static const unsigned long cc_mask[6] = { [0] = X86_EFLAGS_OF, [1] = X86_EFLAGS_CF, [2] = X86_EFLAGS_ZF, @@ -201,15 +201,21 @@ void int3_emulate_jcc(struct pt_regs *re bool match; if (cc < 0xc) { - match = regs->flags & jcc_mask[cc >> 1]; + match = flags & cc_mask[cc >> 1]; } else { - match = ((regs->flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^ - ((regs->flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT); + match = ((flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^ + ((flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT); if (cc >= 0xe) - match = match || (regs->flags & X86_EFLAGS_ZF); + match = match || (flags & X86_EFLAGS_ZF); } - if ((match && !invert) || (!match && invert)) + return (match && !invert) || (!match && invert); +} + +static __always_inline +void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp) +{ + if (__emulate_cc(regs->flags, cc)) ip += disp; int3_emulate_jmp(regs, ip); --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "x86.h" #include "tss.h" @@ -416,31 +417,6 @@ static int fastop(struct x86_emulate_ctx ON64(FOP3E(op##q, rax, rdx, cl)) \ FOP_END -/* Special case for SETcc - 1 instruction per cc */ -#define FOP_SETCC(op) \ - FOP_FUNC(op) \ - #op " %al \n\t" \ - FOP_RET(op) - -FOP_START(setcc) -FOP_SETCC(seto) -FOP_SETCC(setno) -FOP_SETCC(setc) -FOP_SETCC(setnc) -FOP_SETCC(setz) -FOP_SETCC(setnz) -FOP_SETCC(setbe) -FOP_SETCC(setnbe) -FOP_SETCC(sets) -FOP_SETCC(setns) -FOP_SETCC(setp) -FOP_SETCC(setnp) -FOP_SETCC(setl) -FOP_SETCC(setnl) -FOP_SETCC(setle) -FOP_SETCC(setnle) -FOP_END; - FOP_START(salc) FOP_FUNC(salc) "pushf; sbb %al, %al; popf \n\t" @@ -1063,13 +1039,7 @@ static int em_bsr_c(struct x86_emulate_c static __always_inline u8 test_cc(unsigned int condition, unsigned long flags) { - u8 rc; - void (*fop)(void) = (void *)em_setcc + FASTOP_SIZE * (condition & 0xf); - - flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF; - asm("push %[flags]; popf; " CALL_NOSPEC - : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags)); - return rc; + return __emulate_cc(flags, condition & 0xf); } static void fetch_register_operand(struct operand *op) From patchwork Mon Dec 4 09:37:12 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Peter Zijlstra X-Patchwork-Id: 13478005 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="GI9fusMq" Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 9907BD2; Mon, 4 Dec 2023 01:39:56 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=H9F+SG0jcXjCMA+hoF+Z9AplgtYJpt1PN/dsuLjAMTg=; b=GI9fusMqIKIMDgOataLeOA6L29 ELtX3JSwhcqgFDC9EOI33qDJMM4Vur0H0pMdS2WT6/Tfc45bLVdueDVlvK6C8SJHh36tmHIi/t6yh g+epxU5qqvQjgQxvaq6Xk0btkrvgB39T5iTjmJCxEAkpAqqlxulpsi5ljenzWUFJyhTrE3w0V+OjN DrBCDFuJwRn2fObHlA9ZURdFgSDsd1W37iofnA3cNAv6p7NDx36zBwswl158x7ykTzemSzQeqAUUb BheiyAwlrxiGH5U0/ck/BvjhNG2PRfnSPKx9vWKDxQGDue8bcUeOibb2CX/68Hhg1mqdl8CvqWCOx cmDljhyg==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1rA5QZ-000X0c-6w; Mon, 04 Dec 2023 09:39:47 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id AFA29301C36; Mon, 4 Dec 2023 10:39:45 +0100 (CET) Message-Id: <20231204093732.436930429@infradead.org> User-Agent: quilt/0.65 Date: Mon, 04 Dec 2023 10:37:12 +0100 From: Peter Zijlstra To: Sean Christopherson , Paolo Bonzini , Josh Poimboeuf , Thomas Gleixner Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, x86@kernel.org, kvm@vger.kernel.org Subject: [PATCH 10/11] x86/nospec: JMP_NOSPEC References: <20231204093702.989848513@infradead.org> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Signed-off-by: Peter Zijlstra (Intel) --- arch/x86/include/asm/nospec-branch.h | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -403,6 +403,17 @@ static inline void call_depth_return_thu "call *%[thunk_target]\n", \ X86_FEATURE_RETPOLINE_LFENCE) +# define JMP_NOSPEC \ + ALTERNATIVE_2( \ + ANNOTATE_RETPOLINE_SAFE \ + "jmp *%[thunk_target]\n", \ + "jmp __x86_indirect_thunk_%V[thunk_target]\n", \ + X86_FEATURE_RETPOLINE, \ + "lfence;\n" \ + ANNOTATE_RETPOLINE_SAFE \ + "jmp *%[thunk_target]\n", \ + X86_FEATURE_RETPOLINE_LFENCE) + # define THUNK_TARGET(addr) [thunk_target] "r" (addr) #else /* CONFIG_X86_32 */ @@ -433,10 +444,31 @@ static inline void call_depth_return_thu "call *%[thunk_target]\n", \ X86_FEATURE_RETPOLINE_LFENCE) +# define JMP_NOSPEC \ + ALTERNATIVE_2( \ + ANNOTATE_RETPOLINE_SAFE \ + "jmp *%[thunk_target]\n", \ + " jmp 901f;\n" \ + " .align 16\n" \ + "901: call 903f;\n" \ + "902: pause;\n" \ + " lfence;\n" \ + " jmp 902b;\n" \ + " .align 16\n" \ + "903: lea 4(%%esp), %%esp;\n" \ + " pushl %[thunk_target];\n" \ + " ret;\n" \ + X86_FEATURE_RETPOLINE, \ + "lfence;\n" \ + ANNOTATE_RETPOLINE_SAFE \ + "jmp *%[thunk_target]\n", \ + X86_FEATURE_RETPOLINE_LFENCE) + # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) #endif #else /* No retpoline for C / inline asm */ # define CALL_NOSPEC "call *%[thunk_target]\n" +# define JMP_NOSPEC "jmp *%[thunk_target]\n" # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) #endif From patchwork Mon Dec 4 09:37:13 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Peter Zijlstra X-Patchwork-Id: 13478004 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="UJ4QbxuB" Received: from desiato.infradead.org (desiato.infradead.org [IPv6:2001:8b0:10b:1:d65d:64ff:fe57:4e05]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id E3DF8CD; Mon, 4 Dec 2023 01:39:52 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=cKtSfZ5sWaSTj7tZINYGDLIwyP/HVC5Nz5I81xWPCwk=; b=UJ4QbxuBSELUeSrjN4lie1/3Nn sLbJRS71hZLDxuqqfx0RU270C87H+Ymw6e3gZXjYlGd71WgXV0HAq4BtolDaoiqcNJFkHY0dKMTG3 /zeZmKHZGL8lJchX5o+/rizS5A9SMrwY5iIjr8PSuKOAhtNFrTY22O9pFjYU1x33YdR0oVwCWAZjk mXcSzUX5aDu58oFsY3OITPf3xQLg3G6yydcoZcaTi0fHwDai0LrRHajnvymn3gIYbMh78WKnm2UFQ vECPvLWn/QORqhFYQA/SBQJQVoWs3DywzZmFe3jyjv0Kl/1ZZaUMHVjwGQAV5bdgJ92kTeaHa0F5n AgYZ1C6A==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.96 #2 (Red Hat Linux)) id 1rA5QZ-004KSk-1O; Mon, 04 Dec 2023 09:39:47 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id B32B7301C46; Mon, 4 Dec 2023 10:39:45 +0100 (CET) Message-Id: <20231204093732.580299853@infradead.org> User-Agent: quilt/0.65 Date: Mon, 04 Dec 2023 10:37:13 +0100 From: Peter Zijlstra To: Sean Christopherson , Paolo Bonzini , Josh Poimboeuf , Thomas Gleixner Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, x86@kernel.org, kvm@vger.kernel.org Subject: [PATCH 11/11] x86/kvm/emulate: Avoid RET for fastops References: <20231204093702.989848513@infradead.org> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Since there is only a single fastop() function, convert the FASTOP stuff from CALL_NOSPEC+RET to JMP_NOSPEC+JMP, avoiding the return thunks and all that jazz. Specifically FASTOPs rely on the return thunk to preserve EFLAGS, which not all of them can trivially do (call depth tracing suffers here). Objtool strenuously complains about this: - indirect call without a .rodata, fails to determine JUMP_TABLE, annotate - fastop functions fall through, exception - unreachable instruction after fastop_return, save/restore Signed-off-by: Peter Zijlstra (Intel) --- arch/x86/kvm/emulate.c | 20 +++++++++++++++----- include/linux/objtool_types.h | 1 + tools/include/linux/objtool_types.h | 1 + tools/objtool/check.c | 11 ++++++++++- 4 files changed, 27 insertions(+), 6 deletions(-) --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -285,8 +285,8 @@ static void invalidate_registers(struct * different operand sizes can be reached by calculation, rather than a jump * table (which would be bigger than the code). * - * The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for ENDBR - * and 1 for the straight line speculation INT3, leaves 7 bytes for the + * The 16 byte alignment, considering 5 bytes for the JMP, 4 for ENDBR + * and 1 for the straight line speculation INT3, leaves 6 bytes for the * body of the function. Currently none is larger than 4. */ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); @@ -304,7 +304,7 @@ static int fastop(struct x86_emulate_ctx __FOP_FUNC(#name) #define __FOP_RET(name) \ - "11: " ASM_RET \ + "11: jmp fastop_return; int3 \n\t" \ ".size " name ", .-" name "\n\t" #define FOP_RET(name) \ @@ -5071,14 +5071,24 @@ static void fetch_possible_mmx_operand(s kvm_read_mmx_reg(op->addr.mm, &op->mm_val); } -static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop) +/* + * All the FASTOP magic above relies on there being *one* instance of this + * so it can JMP back, avoiding RET and it's various thunks. + */ +static noinline int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop) { ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF; if (!(ctxt->d & ByteOp)) fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE; - asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n" + asm("push %[flags]; popf \n\t" + UNWIND_HINT(UNWIND_HINT_TYPE_SAVE, 0, 0, 0) + ASM_ANNOTATE(ANNOTYPE_JUMP_TABLE) + JMP_NOSPEC + "fastop_return: \n\t" + UNWIND_HINT(UNWIND_HINT_TYPE_RESTORE, 0, 0, 0) + "pushf; pop %[flags]\n" : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags), [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT : "c"(ctxt->src2.val)); --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -64,5 +64,6 @@ struct unwind_hint { #define ANNOTYPE_UNRET_BEGIN 5 #define ANNOTYPE_IGNORE_ALTS 6 #define ANNOTYPE_INTRA_FUNCTION_CALLS 7 +#define ANNOTYPE_JUMP_TABLE 8 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -64,5 +64,6 @@ struct unwind_hint { #define ANNOTYPE_UNRET_BEGIN 5 #define ANNOTYPE_IGNORE_ALTS 6 #define ANNOTYPE_INTRA_FUNCTION_CALLS 7 +#define ANNOTYPE_JUMP_TABLE 8 #endif /* _LINUX_OBJTOOL_TYPES_H */ --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2386,6 +2386,14 @@ static int __annotate_late(struct objtoo insn->unret = 1; break; + /* + * Must be after add_jump_table(); for it doesn't set a sane + * _jump_table value. + */ + case ANNOTYPE_JUMP_TABLE: + insn->_jump_table = (void *)1; + break; + default: break; } @@ -3459,7 +3467,8 @@ static int validate_branch(struct objtoo if (func && insn_func(insn) && func != insn_func(insn)->pfunc) { /* Ignore KCFI type preambles, which always fall through */ if (!strncmp(func->name, "__cfi_", 6) || - !strncmp(func->name, "__pfx_", 6)) + !strncmp(func->name, "__pfx_", 6) || + !strcmp(insn_func(insn)->name, "fastop")) return 0; WARN("%s() falls through to next function %s()",