@@ -67,6 +67,13 @@ struct orc_entry {
#endif
} __packed;
+static inline unsigned long orc_ip(const int *ip)
+{
+ return (unsigned long)ip + *ip;
+}
+
+void orc_sort(int *ip_table, struct orc_entry *orc_table, u32 num_orcs);
+
#endif /* __ASSEMBLY__ */
#endif /* _ORC_TYPES_H */
@@ -25,11 +25,6 @@ extern struct orc_entry __stop_orc_unwind[];
static bool orc_init __ro_after_init;
static unsigned int lookup_num_blocks __ro_after_init;
-static inline unsigned long orc_ip(const int *ip)
-{
- return (unsigned long)ip + *ip;
-}
-
static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table,
unsigned int num_entries, unsigned long ip)
{
@@ -188,53 +183,6 @@ static struct orc_entry *orc_find(unsigned long ip)
}
#ifdef CONFIG_MODULES
-
-static DEFINE_MUTEX(sort_mutex);
-static int *cur_orc_ip_table = __start_orc_unwind_ip;
-static struct orc_entry *cur_orc_table = __start_orc_unwind;
-
-static void orc_sort_swap(void *_a, void *_b, int size)
-{
- struct orc_entry *orc_a, *orc_b;
- struct orc_entry orc_tmp;
- int *a = _a, *b = _b, tmp;
- int delta = _b - _a;
-
- /* Swap the .orc_unwind_ip entries: */
- tmp = *a;
- *a = *b + delta;
- *b = tmp - delta;
-
- /* Swap the corresponding .orc_unwind entries: */
- orc_a = cur_orc_table + (a - cur_orc_ip_table);
- orc_b = cur_orc_table + (b - cur_orc_ip_table);
- orc_tmp = *orc_a;
- *orc_a = *orc_b;
- *orc_b = orc_tmp;
-}
-
-static int orc_sort_cmp(const void *_a, const void *_b)
-{
- struct orc_entry *orc_a;
- const int *a = _a, *b = _b;
- unsigned long a_val = orc_ip(a);
- unsigned long b_val = orc_ip(b);
-
- if (a_val > b_val)
- return 1;
- if (a_val < b_val)
- return -1;
-
- /*
- * The "weak" section terminator entries need to always be on the left
- * to ensure the lookup code skips them in favor of real entries.
- * These terminator entries exist to handle any gaps created by
- * whitelisted .o files which didn't get objtool generation.
- */
- orc_a = cur_orc_table + (a - cur_orc_ip_table);
- return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1;
-}
-
void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size,
void *_orc, size_t orc_size)
{
@@ -246,16 +194,7 @@ void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size,
orc_size % sizeof(*orc) != 0 ||
num_entries != orc_size / sizeof(*orc));
- /*
- * The 'cur_orc_*' globals allow the orc_sort_swap() callback to
- * associate an .orc_unwind_ip table entry with its corresponding
- * .orc_unwind entry so they can both be swapped.
- */
- mutex_lock(&sort_mutex);
- cur_orc_ip_table = orc_ip;
- cur_orc_table = orc;
- sort(orc_ip, num_entries, sizeof(int), orc_sort_cmp, orc_sort_swap);
- mutex_unlock(&sort_mutex);
+ orc_sort(orc_ip, orc, num_entries);
mod->arch.orc_unwind_ip = orc_ip;
mod->arch.orc_unwind = orc;
@@ -50,6 +50,7 @@ lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o
lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
lib-$(CONFIG_RETPOLINE) += retpoline.o
+lib-$(CONFIG_UNWINDER_ORC) += orc.o
obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
obj-y += iomem.o
new file mode 100644
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ORC sorting shared by the compressed boot code and ORC module
+ * support.
+ */
+
+#include <asm/orc_types.h>
+#include <linux/mutex.h>
+#include <linux/sort.h>
+
+#ifndef ORC_COMPRESSED_BOOT
+static DEFINE_MUTEX(sort_mutex);
+
+#define sort_mutex_lock() mutex_lock(&sort_mutex)
+#define sort_mutex_unlock() mutex_unlock(&sort_mutex)
+#else /* ORC_COMPRESSED_BOOT */
+#define sort_mutex_lock()
+#define sort_mutex_unlock()
+#endif /* ORC_COMPRESSED_BOOT */
+
+static int *cur_orc_ip_table;
+static struct orc_entry *cur_orc_table;
+
+static void orc_sort_swap(void *_a, void *_b, int size)
+{
+ struct orc_entry *orc_a, *orc_b;
+ int *a = _a, *b = _b, tmp;
+ int delta = _b - _a;
+
+ /* Swap the .orc_unwind_ip entries: */
+ tmp = *a;
+ *a = *b + delta;
+ *b = tmp - delta;
+
+ /* Swap the corresponding .orc_unwind entries: */
+ orc_a = cur_orc_table + (a - cur_orc_ip_table);
+ orc_b = cur_orc_table + (b - cur_orc_ip_table);
+ swap(*orc_a, *orc_b);
+}
+
+static int orc_sort_cmp(const void *_a, const void *_b)
+{
+ const int *a = _a, *b = _b;
+ unsigned long a_val = orc_ip(a);
+ unsigned long b_val = orc_ip(b);
+ struct orc_entry *orc_a;
+
+ if (a_val > b_val)
+ return 1;
+ if (a_val < b_val)
+ return -1;
+
+ /*
+ * The "weak" section terminator entries need to always be on the left
+ * to ensure the lookup code skips them in favor of real entries.
+ * These terminator entries exist to handle any gaps created by
+ * whitelisted .o files which didn't get objtool generation.
+ */
+ orc_a = cur_orc_table + (a - cur_orc_ip_table);
+ return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1;
+}
+
+void orc_sort(int *ip_table, struct orc_entry *orc_table, u32 num_orcs)
+{
+ /*
+ * The 'cur_orc_*' globals allow the orc_sort_swap() callback to
+ * associate an .orc_unwind_ip table entry with its corresponding
+ * .orc_unwind entry so they can both be swapped.
+ */
+ sort_mutex_lock();
+ cur_orc_ip_table = ip_table;
+ cur_orc_table = orc_table;
+ sort(ip_table, num_orcs, sizeof(int), orc_sort_cmp,
+ orc_sort_swap);
+ sort_mutex_unlock();
+}
@@ -96,11 +96,6 @@ struct orc_entry *g_orc_table;
pthread_t orc_sort_thread;
-static inline unsigned long orc_ip(const int *ip)
-{
- return (unsigned long)ip + *ip;
-}
-
static int orc_sort_cmp(const void *_a, const void *_b)
{
struct orc_entry *orc_a;
@@ -67,6 +67,13 @@ struct orc_entry {
#endif
} __packed;
+static inline unsigned long orc_ip(const int *ip)
+{
+ return (unsigned long)ip + *ip;
+}
+
+void orc_sort(int *ip_table, struct orc_entry *orc_table, u32 num_orcs);
+
#endif /* __ASSEMBLY__ */
#endif /* _ORC_TYPES_H */
In order to be able to sort ORC entries from both the kernel and the pre-boot compressed environment, place ORC sorting function into the new file arch/x86/lib/orc.c. It can be then included directly by the pre-boot code, and ORC_COMPRESSED_BOOT definition guards out the sort_mutex which is unavailable and unneeded in that environment. Placing orc_ip() into `include/asm/orc_types.h` wipes out sorttable.h's little code dup. Suggested-by: Peter Zijlstra <peterz@infradead.org> Signed-off-by: Alexander Lobakin <alexandr.lobakin@intel.com> --- arch/x86/include/asm/orc_types.h | 7 +++ arch/x86/kernel/unwind_orc.c | 63 +-------------------- arch/x86/lib/Makefile | 1 + arch/x86/lib/orc.c | 76 ++++++++++++++++++++++++++ scripts/sorttable.h | 5 -- tools/arch/x86/include/asm/orc_types.h | 7 +++ 6 files changed, 92 insertions(+), 67 deletions(-) create mode 100644 arch/x86/lib/orc.c