deleted file mode 120000
@@ -1 +0,0 @@
-../../../arch/arm64/kernel/vdso
\ No newline at end of file
deleted file mode 120000
@@ -1 +0,0 @@
-../../../arch/loongarch/vdso
\ No newline at end of file
deleted file mode 120000
@@ -1 +0,0 @@
-../../../arch/powerpc/kernel/vdso
\ No newline at end of file
deleted file mode 120000
@@ -1 +0,0 @@
-../../../arch/s390/kernel/vdso64
\ No newline at end of file
deleted file mode 120000
@@ -1 +0,0 @@
-../../../arch/x86/entry/vdso/
\ No newline at end of file
new file mode 100644
@@ -0,0 +1,31 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+# description: ftrace - function profiler with function graph tracing
+# requires: function_profile_enabled set_ftrace_filter function_graph:tracer
+
+# The function graph tracer can now be run along side of the function
+# profiler. But there was a bug that caused the combination of the two
+# to crash. It also required the function graph tracer to be started
+# first.
+#
+# This test triggers that bug
+#
+# We need both function_graph and profiling to run this test
+
+fail() { # mesg
+ echo $1
+ exit_fail
+}
+
+echo "Enabling function graph tracer:"
+echo function_graph > current_tracer
+echo "enable profiler"
+
+# Older kernels do not allow function_profile to be enabled with
+# function graph tracer. If the below fails, mark it as unsupported
+echo 1 > function_profile_enabled || exit_unsupported
+
+# Let it run for a bit to make sure nothing explodes
+sleep 1
+
+exit 0
@@ -60,12 +60,6 @@ unsigned int rseq_size = -1U;
/* Flags used during rseq registration. */
unsigned int rseq_flags;
-/*
- * rseq feature size supported by the kernel. 0 if the registration was
- * unsuccessful.
- */
-unsigned int rseq_feature_size = -1U;
-
static int rseq_ownership;
static int rseq_reg_success; /* At least one rseq registration has succeded. */
@@ -111,6 +105,43 @@ int rseq_available(void)
}
}
+/* The rseq areas need to be at least 32 bytes. */
+static
+unsigned int get_rseq_min_alloc_size(void)
+{
+ unsigned int alloc_size = rseq_size;
+
+ if (alloc_size < ORIG_RSEQ_ALLOC_SIZE)
+ alloc_size = ORIG_RSEQ_ALLOC_SIZE;
+ return alloc_size;
+}
+
+/*
+ * Return the feature size supported by the kernel.
+ *
+ * Depending on the value returned by getauxval(AT_RSEQ_FEATURE_SIZE):
+ *
+ * 0: Return ORIG_RSEQ_FEATURE_SIZE (20)
+ * > 0: Return the value from getauxval(AT_RSEQ_FEATURE_SIZE).
+ *
+ * It should never return a value below ORIG_RSEQ_FEATURE_SIZE.
+ */
+static
+unsigned int get_rseq_kernel_feature_size(void)
+{
+ unsigned long auxv_rseq_feature_size, auxv_rseq_align;
+
+ auxv_rseq_align = getauxval(AT_RSEQ_ALIGN);
+ assert(!auxv_rseq_align || auxv_rseq_align <= RSEQ_THREAD_AREA_ALLOC_SIZE);
+
+ auxv_rseq_feature_size = getauxval(AT_RSEQ_FEATURE_SIZE);
+ assert(!auxv_rseq_feature_size || auxv_rseq_feature_size <= RSEQ_THREAD_AREA_ALLOC_SIZE);
+ if (auxv_rseq_feature_size)
+ return auxv_rseq_feature_size;
+ else
+ return ORIG_RSEQ_FEATURE_SIZE;
+}
+
int rseq_register_current_thread(void)
{
int rc;
@@ -119,7 +150,7 @@ int rseq_register_current_thread(void)
/* Treat libc's ownership as a successful registration. */
return 0;
}
- rc = sys_rseq(&__rseq_abi, rseq_size, 0, RSEQ_SIG);
+ rc = sys_rseq(&__rseq_abi, get_rseq_min_alloc_size(), 0, RSEQ_SIG);
if (rc) {
if (RSEQ_READ_ONCE(rseq_reg_success)) {
/* Incoherent success/failure within process. */
@@ -140,28 +171,12 @@ int rseq_unregister_current_thread(void)
/* Treat libc's ownership as a successful unregistration. */
return 0;
}
- rc = sys_rseq(&__rseq_abi, rseq_size, RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG);
+ rc = sys_rseq(&__rseq_abi, get_rseq_min_alloc_size(), RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG);
if (rc)
return -1;
return 0;
}
-static
-unsigned int get_rseq_feature_size(void)
-{
- unsigned long auxv_rseq_feature_size, auxv_rseq_align;
-
- auxv_rseq_align = getauxval(AT_RSEQ_ALIGN);
- assert(!auxv_rseq_align || auxv_rseq_align <= RSEQ_THREAD_AREA_ALLOC_SIZE);
-
- auxv_rseq_feature_size = getauxval(AT_RSEQ_FEATURE_SIZE);
- assert(!auxv_rseq_feature_size || auxv_rseq_feature_size <= RSEQ_THREAD_AREA_ALLOC_SIZE);
- if (auxv_rseq_feature_size)
- return auxv_rseq_feature_size;
- else
- return ORIG_RSEQ_FEATURE_SIZE;
-}
-
static __attribute__((constructor))
void rseq_init(void)
{
@@ -178,28 +193,54 @@ void rseq_init(void)
}
if (libc_rseq_size_p && libc_rseq_offset_p && libc_rseq_flags_p &&
*libc_rseq_size_p != 0) {
+ unsigned int libc_rseq_size;
+
/* rseq registration owned by glibc */
rseq_offset = *libc_rseq_offset_p;
- rseq_size = *libc_rseq_size_p;
+ libc_rseq_size = *libc_rseq_size_p;
rseq_flags = *libc_rseq_flags_p;
- rseq_feature_size = get_rseq_feature_size();
- if (rseq_feature_size > rseq_size)
- rseq_feature_size = rseq_size;
+
+ /*
+ * Previous versions of glibc expose the value
+ * 32 even though the kernel only supported 20
+ * bytes initially. Therefore treat 32 as a
+ * special-case. glibc 2.40 exposes a 20 bytes
+ * __rseq_size without using getauxval(3) to
+ * query the supported size, while still allocating a 32
+ * bytes area. Also treat 20 as a special-case.
+ *
+ * Special-cases are handled by using the following
+ * value as active feature set size:
+ *
+ * rseq_size = min(32, get_rseq_kernel_feature_size())
+ */
+ switch (libc_rseq_size) {
+ case ORIG_RSEQ_FEATURE_SIZE:
+ fallthrough;
+ case ORIG_RSEQ_ALLOC_SIZE:
+ {
+ unsigned int rseq_kernel_feature_size = get_rseq_kernel_feature_size();
+
+ if (rseq_kernel_feature_size < ORIG_RSEQ_ALLOC_SIZE)
+ rseq_size = rseq_kernel_feature_size;
+ else
+ rseq_size = ORIG_RSEQ_ALLOC_SIZE;
+ break;
+ }
+ default:
+ /* Otherwise just use the __rseq_size from libc as rseq_size. */
+ rseq_size = libc_rseq_size;
+ break;
+ }
return;
}
rseq_ownership = 1;
if (!rseq_available()) {
rseq_size = 0;
- rseq_feature_size = 0;
return;
}
rseq_offset = (void *)&__rseq_abi - rseq_thread_pointer();
rseq_flags = 0;
- rseq_feature_size = get_rseq_feature_size();
- if (rseq_feature_size == ORIG_RSEQ_FEATURE_SIZE)
- rseq_size = ORIG_RSEQ_ALLOC_SIZE;
- else
- rseq_size = RSEQ_THREAD_AREA_ALLOC_SIZE;
}
static __attribute__((destructor))
@@ -209,7 +250,6 @@ void rseq_exit(void)
return;
rseq_offset = 0;
rseq_size = -1U;
- rseq_feature_size = -1U;
rseq_ownership = 0;
}
@@ -68,12 +68,6 @@ extern unsigned int rseq_size;
/* Flags used during rseq registration. */
extern unsigned int rseq_flags;
-/*
- * rseq feature size supported by the kernel. 0 if the registration was
- * unsuccessful.
- */
-extern unsigned int rseq_feature_size;
-
enum rseq_mo {
RSEQ_MO_RELAXED = 0,
RSEQ_MO_CONSUME = 1, /* Unused */
@@ -193,7 +187,7 @@ static inline uint32_t rseq_current_cpu(void)
static inline bool rseq_node_id_available(void)
{
- return (int) rseq_feature_size >= rseq_offsetofend(struct rseq_abi, node_id);
+ return (int) rseq_size >= rseq_offsetofend(struct rseq_abi, node_id);
}
/*
@@ -207,7 +201,7 @@ static inline uint32_t rseq_current_node_id(void)
static inline bool rseq_mm_cid_available(void)
{
- return (int) rseq_feature_size >= rseq_offsetofend(struct rseq_abi, mm_cid);
+ return (int) rseq_size >= rseq_offsetofend(struct rseq_abi, mm_cid);
}
static inline uint32_t rseq_current_mm_cid(void)
@@ -9,10 +9,8 @@ ifeq ($(ARCH),$(filter $(ARCH),x86 x86_64))
TEST_GEN_PROGS += vdso_standalone_test_x86
endif
TEST_GEN_PROGS += vdso_test_correctness
-ifeq ($(ARCH)$(CONFIG_X86_32),$(filter $(ARCH)$(CONFIG_X86_32),x86 x86_64 loongarch arm64 powerpc s390))
TEST_GEN_PROGS += vdso_test_getrandom
TEST_GEN_PROGS += vdso_test_chacha
-endif
CFLAGS := -std=gnu99 -O2
@@ -37,9 +35,9 @@ $(OUTPUT)/vdso_test_getrandom: CFLAGS += -isystem $(top_srcdir)/tools/include \
$(KHDR_INCLUDES) \
-isystem $(top_srcdir)/include/uapi
-$(OUTPUT)/vdso_test_chacha: $(top_srcdir)/tools/arch/$(SRCARCH)/vdso/vgetrandom-chacha.S
+$(OUTPUT)/vdso_test_chacha: vgetrandom-chacha.S
$(OUTPUT)/vdso_test_chacha: CFLAGS += -idirafter $(top_srcdir)/tools/include \
-idirafter $(top_srcdir)/tools/include/generated \
-idirafter $(top_srcdir)/arch/$(SRCARCH)/include \
-idirafter $(top_srcdir)/include \
- -D__ASSEMBLY__ -Wa,--noexecstack
+ -Wa,--noexecstack
@@ -3,6 +3,7 @@
* Copyright (C) 2022-2024 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*/
+#include <linux/compiler.h>
#include <tools/le_byteshift.h>
#include <sys/random.h>
#include <sys/auxv.h>
@@ -73,10 +74,10 @@ static void reference_chacha20_blocks(uint8_t *dst_bytes, const uint32_t *key, u
counter[1] = s[13];
}
-typedef uint8_t u8;
-typedef uint32_t u32;
-typedef uint64_t u64;
-#include <vdso/getrandom.h>
+void __weak __arch_chacha20_blocks_nostack(uint8_t *dst_bytes, const uint32_t *key, uint32_t *counter, size_t nblocks)
+{
+ ksft_exit_skip("Not implemented on architecture\n");
+}
int main(int argc, char *argv[])
{
@@ -90,10 +91,8 @@ int main(int argc, char *argv[])
ksft_set_plan(1);
for (unsigned int trial = 0; trial < TRIALS; ++trial) {
- if (getrandom(key, sizeof(key), 0) != sizeof(key)) {
- printf("getrandom() failed!\n");
- return KSFT_SKIP;
- }
+ if (getrandom(key, sizeof(key), 0) != sizeof(key))
+ ksft_exit_skip("getrandom() failed unexpectedly\n");
memset(counter1, 0, sizeof(counter1));
reference_chacha20_blocks(output1, key, counter1, BLOCKS);
for (unsigned int split = 0; split < BLOCKS; ++split) {
@@ -102,8 +101,10 @@ int main(int argc, char *argv[])
if (split)
__arch_chacha20_blocks_nostack(output2, key, counter2, split);
__arch_chacha20_blocks_nostack(output2 + split * BLOCK_SIZE, key, counter2, BLOCKS - split);
- if (memcmp(output1, output2, sizeof(output1)) || memcmp(counter1, counter2, sizeof(counter1)))
- return KSFT_FAIL;
+ if (memcmp(output1, output2, sizeof(output1)))
+ ksft_exit_fail_msg("Main loop outputs do not match on trial %u, split %u\n", trial, split);
+ if (memcmp(counter1, counter2, sizeof(counter1)))
+ ksft_exit_fail_msg("Main loop counters do not match on trial %u, split %u\n", trial, split);
}
}
memset(counter1, 0, sizeof(counter1));
@@ -113,14 +114,19 @@ int main(int argc, char *argv[])
reference_chacha20_blocks(output1, key, counter1, BLOCKS);
__arch_chacha20_blocks_nostack(output2, key, counter2, BLOCKS);
- if (memcmp(output1, output2, sizeof(output1)) || memcmp(counter1, counter2, sizeof(counter1)))
- return KSFT_FAIL;
+ if (memcmp(output1, output2, sizeof(output1)))
+ ksft_exit_fail_msg("Block limit outputs do not match after first round\n");
+ if (memcmp(counter1, counter2, sizeof(counter1)))
+ ksft_exit_fail_msg("Block limit counters do not match after first round\n");
reference_chacha20_blocks(output1, key, counter1, BLOCKS);
__arch_chacha20_blocks_nostack(output2, key, counter2, BLOCKS);
- if (memcmp(output1, output2, sizeof(output1)) || memcmp(counter1, counter2, sizeof(counter1)))
- return KSFT_FAIL;
+ if (memcmp(output1, output2, sizeof(output1)))
+ ksft_exit_fail_msg("Block limit outputs do not match after second round\n");
+ if (memcmp(counter1, counter2, sizeof(counter1)))
+ ksft_exit_fail_msg("Block limit counters do not match after second round\n");
ksft_test_result_pass("chacha: PASS\n");
- return KSFT_PASS;
+ ksft_exit_pass();
+ return 0;
}
@@ -11,6 +11,7 @@
#include <string.h>
#include <time.h>
#include <unistd.h>
+#include <sched.h>
#include <signal.h>
#include <sys/auxv.h>
#include <sys/mman.h>
@@ -40,6 +41,9 @@
} while (0)
#endif
+#define ksft_assert(condition) \
+ do { if (!(condition)) ksft_exit_fail_msg("Assertion failed: %s\n", #condition); } while (0)
+
static struct {
pthread_mutex_t lock;
void **states;
@@ -111,26 +115,19 @@ static void vgetrandom_init(void)
const char *version = versions[VDSO_VERSION];
const char *name = names[VDSO_NAMES][6];
unsigned long sysinfo_ehdr = getauxval(AT_SYSINFO_EHDR);
- size_t ret;
+ ssize_t ret;
- if (!sysinfo_ehdr) {
- printf("AT_SYSINFO_EHDR is not present!\n");
- exit(KSFT_SKIP);
- }
+ if (!sysinfo_ehdr)
+ ksft_exit_skip("AT_SYSINFO_EHDR is not present\n");
vdso_init_from_sysinfo_ehdr(sysinfo_ehdr);
vgrnd.fn = (__typeof__(vgrnd.fn))vdso_sym(version, name);
- if (!vgrnd.fn) {
- printf("%s is missing!\n", name);
- exit(KSFT_FAIL);
- }
+ if (!vgrnd.fn)
+ ksft_exit_skip("%s@%s symbol is missing from vDSO\n", name, version);
ret = VDSO_CALL(vgrnd.fn, 5, NULL, 0, 0, &vgrnd.params, ~0UL);
- if (ret == -ENOSYS) {
- printf("unsupported architecture\n");
- exit(KSFT_SKIP);
- } else if (ret) {
- printf("failed to fetch vgetrandom params!\n");
- exit(KSFT_FAIL);
- }
+ if (ret == -ENOSYS)
+ ksft_exit_skip("CPU does not have runtime support\n");
+ else if (ret)
+ ksft_exit_fail_msg("Failed to fetch vgetrandom params: %zd\n", ret);
}
static ssize_t vgetrandom(void *buf, size_t len, unsigned long flags)
@@ -139,10 +136,7 @@ static ssize_t vgetrandom(void *buf, size_t len, unsigned long flags)
if (!state) {
state = vgetrandom_get_state();
- if (!state) {
- printf("vgetrandom_get_state failed!\n");
- exit(KSFT_FAIL);
- }
+ ksft_assert(state);
}
return VDSO_CALL(vgrnd.fn, 5, buf, len, flags, state, vgrnd.params.size_of_opaque_state);
}
@@ -154,7 +148,7 @@ static void *test_vdso_getrandom(void *ctx)
for (size_t i = 0; i < TRIALS; ++i) {
unsigned int val;
ssize_t ret = vgetrandom(&val, sizeof(val), 0);
- assert(ret == sizeof(val));
+ ksft_assert(ret == sizeof(val));
}
return NULL;
}
@@ -164,7 +158,7 @@ static void *test_libc_getrandom(void *ctx)
for (size_t i = 0; i < TRIALS; ++i) {
unsigned int val;
ssize_t ret = getrandom(&val, sizeof(val), 0);
- assert(ret == sizeof(val));
+ ksft_assert(ret == sizeof(val));
}
return NULL;
}
@@ -174,7 +168,7 @@ static void *test_syscall_getrandom(void *ctx)
for (size_t i = 0; i < TRIALS; ++i) {
unsigned int val;
ssize_t ret = syscall(__NR_getrandom, &val, sizeof(val), 0);
- assert(ret == sizeof(val));
+ ksft_assert(ret == sizeof(val));
}
return NULL;
}
@@ -209,7 +203,7 @@ static void bench_multi(void)
clock_gettime(CLOCK_MONOTONIC, &start);
for (size_t i = 0; i < THREADS; ++i)
- assert(pthread_create(&threads[i], NULL, test_vdso_getrandom, NULL) == 0);
+ ksft_assert(pthread_create(&threads[i], NULL, test_vdso_getrandom, NULL) == 0);
for (size_t i = 0; i < THREADS; ++i)
pthread_join(threads[i], NULL);
clock_gettime(CLOCK_MONOTONIC, &end);
@@ -218,7 +212,7 @@ static void bench_multi(void)
clock_gettime(CLOCK_MONOTONIC, &start);
for (size_t i = 0; i < THREADS; ++i)
- assert(pthread_create(&threads[i], NULL, test_libc_getrandom, NULL) == 0);
+ ksft_assert(pthread_create(&threads[i], NULL, test_libc_getrandom, NULL) == 0);
for (size_t i = 0; i < THREADS; ++i)
pthread_join(threads[i], NULL);
clock_gettime(CLOCK_MONOTONIC, &end);
@@ -227,7 +221,7 @@ static void bench_multi(void)
clock_gettime(CLOCK_MONOTONIC, &start);
for (size_t i = 0; i < THREADS; ++i)
- assert(pthread_create(&threads[i], NULL, test_syscall_getrandom, NULL) == 0);
+ ksft_assert(pthread_create(&threads[i], NULL, test_syscall_getrandom, NULL) == 0);
for (size_t i = 0; i < THREADS; ++i)
pthread_join(threads[i], NULL);
clock_gettime(CLOCK_MONOTONIC, &end);
@@ -252,48 +246,46 @@ static void kselftest(void)
for (size_t i = 0; i < 1000; ++i) {
ssize_t ret = vgetrandom(weird_size, sizeof(weird_size), 0);
- if (ret != sizeof(weird_size))
- exit(KSFT_FAIL);
+ ksft_assert(ret == sizeof(weird_size));
}
ksft_test_result_pass("getrandom: PASS\n");
unshare(CLONE_NEWUSER);
- assert(unshare(CLONE_NEWTIME) == 0);
+ ksft_assert(unshare(CLONE_NEWTIME) == 0);
child = fork();
- assert(child >= 0);
+ ksft_assert(child >= 0);
if (!child) {
vgetrandom_init();
child = getpid();
- assert(ptrace(PTRACE_TRACEME, 0, NULL, NULL) == 0);
- assert(kill(child, SIGSTOP) == 0);
- assert(vgetrandom(weird_size, sizeof(weird_size), 0) == sizeof(weird_size));
+ ksft_assert(ptrace(PTRACE_TRACEME, 0, NULL, NULL) == 0);
+ ksft_assert(kill(child, SIGSTOP) == 0);
+ ksft_assert(vgetrandom(weird_size, sizeof(weird_size), 0) == sizeof(weird_size));
_exit(0);
}
for (;;) {
struct ptrace_syscall_info info = { 0 };
int status, ret;
- assert(waitpid(child, &status, 0) >= 0);
+ ksft_assert(waitpid(child, &status, 0) >= 0);
if (WIFEXITED(status)) {
- if (WEXITSTATUS(status) != 0)
- exit(KSFT_FAIL);
+ ksft_assert(WEXITSTATUS(status) == 0);
break;
}
- assert(WIFSTOPPED(status));
+ ksft_assert(WIFSTOPPED(status));
if (WSTOPSIG(status) == SIGSTOP)
- assert(ptrace(PTRACE_SETOPTIONS, child, 0, PTRACE_O_TRACESYSGOOD) == 0);
+ ksft_assert(ptrace(PTRACE_SETOPTIONS, child, 0, PTRACE_O_TRACESYSGOOD) == 0);
else if (WSTOPSIG(status) == (SIGTRAP | 0x80)) {
- assert(ptrace(PTRACE_GET_SYSCALL_INFO, child, sizeof(info), &info) > 0);
+ ksft_assert(ptrace(PTRACE_GET_SYSCALL_INFO, child, sizeof(info), &info) > 0);
if (info.op == PTRACE_SYSCALL_INFO_ENTRY && info.entry.nr == __NR_getrandom &&
info.entry.args[0] == (uintptr_t)weird_size && info.entry.args[1] == sizeof(weird_size))
- exit(KSFT_FAIL);
+ ksft_exit_fail_msg("vgetrandom passed buffer to syscall getrandom unexpectedly\n");
}
- assert(ptrace(PTRACE_SYSCALL, child, 0, 0) == 0);
+ ksft_assert(ptrace(PTRACE_SYSCALL, child, 0, 0) == 0);
}
ksft_test_result_pass("getrandom timens: PASS\n");
- exit(KSFT_PASS);
+ ksft_exit_pass();
}
static void usage(const char *argv0)
new file mode 100644
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2024 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#define __ASSEMBLY__
+
+#if defined(__aarch64__)
+#include "../../../../arch/arm64/kernel/vdso/vgetrandom-chacha.S"
+#elif defined(__loongarch__)
+#include "../../../../arch/loongarch/vdso/vgetrandom-chacha.S"
+#elif defined(__powerpc__) || defined(__powerpc64__)
+#include "../../../../arch/powerpc/kernel/vdso/vgetrandom-chacha.S"
+#elif defined(__s390x__)
+#include "../../../../arch/s390/kernel/vdso64/vgetrandom-chacha.S"
+#elif defined(__x86_64__)
+#include "../../../../arch/x86/entry/vdso/vgetrandom-chacha.S"
+#endif