@@ -447,6 +447,7 @@ meson=""
ninja=""
skip_meson=no
gettext=""
+mirror_jit="no"
bogus_os="no"
malloc_trim="auto"
@@ -5715,6 +5716,14 @@ if test "$mingw32" = "yes" ; then
done
fi
+# Enable mirror mapping if debug_tcg on supported platform
+
+if test "$debug_tcg" = "yes"; then
+ if test "$linux" = "yes"; then
+ mirror_jit=yes
+ fi
+fi
+
# We can only support ivshmem if we have eventfd
if [ "$eventfd" = "yes" ]; then
ivshmem=yes
@@ -6801,6 +6810,10 @@ if test "$safe_stack" = "yes"; then
echo "CONFIG_SAFESTACK=y" >> $config_host_mak
fi
+if test "$mirror_jit" = "yes"; then
+ echo "CONFIG_MIRROR_JIT=y" >> $config_host_mak
+fi
+
# If we're using a separate build tree, set it up now.
# DIRS are directories which we simply mkdir in the build tree;
# LINKS are things to symlink back into the source tree
@@ -2162,6 +2162,7 @@ endif
summary_info += {'thread sanitizer': config_host.has_key('CONFIG_TSAN')}
summary_info += {'rng-none': config_host.has_key('CONFIG_RNG_NONE')}
summary_info += {'Linux keyring': config_host.has_key('CONFIG_SECRET_KEYRING')}
+summary_info += {'mirror JIT': config_host.has_key('CONFIG_MIRROR_JIT')}
summary(summary_info, bool_yn: true)
if not supported_cpus.contains(cpu)
@@ -519,6 +519,14 @@ struct TranslationBlock {
uintptr_t jmp_list_head;
uintptr_t jmp_list_next[2];
uintptr_t jmp_dest[2];
+
+#if defined(CONFIG_MIRROR_JIT)
+ /*
+ * Store difference to writable mirror
+ * We need this when patching the jump instructions
+ */
+ ptrdiff_t code_rw_mirror_diff;
+#endif
};
extern bool parallel_cpus;
@@ -627,6 +627,10 @@ struct TCGContext {
size_t code_gen_buffer_size;
void *code_gen_ptr;
void *data_gen_ptr;
+#if defined(CONFIG_MIRROR_JIT)
+ int code_gen_buffer_fd;
+ ptrdiff_t code_rw_mirror_diff;
+#endif
/* Threshold to flush the translated code buffer. */
void *code_gen_highwater;
@@ -677,6 +681,20 @@ struct TCGContext {
target_ulong gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS];
};
+static inline void *tcg_mirror_ptr_rw(TCGContext *s, const void *rx)
+{
+#if defined(CONFIG_MIRROR_JIT)
+ return (void *)rx + s->code_rw_mirror_diff;
+#else
+ return (void *)rx;
+#endif
+}
+
+static inline tcg_insn_unit *tcg_code_ptr_rw(TCGContext *s, const void *rx)
+{
+ return (tcg_insn_unit *)tcg_mirror_ptr_rw(s, rx);
+}
+
extern TCGContext tcg_init_ctx;
extern __thread TCGContext *tcg_ctx;
extern TCGv_env cpu_env;
@@ -354,7 +354,12 @@ void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr)
if (TCG_TARGET_HAS_direct_jump) {
uintptr_t offset = tb->jmp_target_arg[n];
uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr;
- tb_target_set_jmp_target(tc_ptr, tc_ptr + offset, addr);
+#if defined(CONFIG_MIRROR_JIT)
+ uintptr_t wr_addr = tc_ptr + offset + tb->code_rw_mirror_diff;
+#else
+ uintptr_t wr_addr = tc_ptr + offset;
+#endif
+ tb_target_set_jmp_target(tc_ptr, tc_ptr + offset, addr, wr_addr);
} else {
tb->jmp_target_arg[n] = addr;
}
@@ -59,6 +59,7 @@
#include "sysemu/cpus.h"
#include "sysemu/cpu-timers.h"
#include "sysemu/tcg.h"
+#include "qemu/memfd.h"
/* #define DEBUG_TB_INVALIDATE */
/* #define DEBUG_TB_FLUSH */
@@ -302,10 +303,12 @@ static target_long decode_sleb128(uint8_t **pp)
static int encode_search(TranslationBlock *tb, uint8_t *block)
{
- uint8_t *highwater = tcg_ctx->code_gen_highwater;
- uint8_t *p = block;
+ const uint8_t *highwater, *start;
+ uint8_t *p;
int i, j, n;
+ highwater = tcg_mirror_ptr_rw(tcg_ctx, tcg_ctx->code_gen_highwater);
+ start = p = tcg_mirror_ptr_rw(tcg_ctx, block);
for (i = 0, n = tb->icount; i < n; ++i) {
target_ulong prev;
@@ -329,7 +332,7 @@ static int encode_search(TranslationBlock *tb, uint8_t *block)
}
}
- return p - block;
+ return p - start;
}
/* The cpu state corresponding to 'searched_pc' is restored.
@@ -1067,12 +1070,29 @@ static inline void *alloc_code_gen_buffer(void)
#else
static inline void *alloc_code_gen_buffer(void)
{
- int prot = PROT_WRITE | PROT_READ | PROT_EXEC;
- int flags = MAP_PRIVATE | MAP_ANONYMOUS;
+ int prot = PROT_READ | PROT_EXEC;
+ int flags = 0;
size_t size = tcg_ctx->code_gen_buffer_size;
+ int fd = -1;
void *buf;
- buf = mmap(NULL, size, prot, flags, -1, 0);
+#if defined(CONFIG_MIRROR_JIT)
+#if defined(CONFIG_LINUX)
+ fd = qemu_memfd_create("tcg-jit", size, false, 0, 0, NULL);
+ if (fd < 0) {
+ return NULL;
+ }
+ tcg_ctx->code_gen_buffer_fd = fd;
+ flags |= MAP_SHARED;
+#else /* defined(CONFIG_LINUX) */
+#error "Mirror JIT unimplemented for this platform."
+#endif /* defined(CONFIG_LINUX) */
+#else /* defined(CONFIG_MIRROR_JIT) */
+ prot |= PROT_WRITE;
+ flags |= MAP_ANONYMOUS | MAP_PRIVATE;
+#endif /* defined(CONFIG_MIRROR_JIT) */
+
+ buf = mmap(NULL, size, prot, flags, fd, 0);
if (buf == MAP_FAILED) {
return NULL;
}
@@ -1118,7 +1138,28 @@ static inline void *alloc_code_gen_buffer(void)
}
#endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
-static inline void code_gen_alloc(size_t tb_size)
+#if defined(CONFIG_MIRROR_JIT)
+#if defined(CONFIG_LINUX)
+static inline void *alloc_jit_rw_mirror(void)
+{
+ int fd = tcg_ctx->code_gen_buffer_fd;
+ size_t size = tcg_ctx->code_gen_buffer_size;
+ void *buf;
+
+ assert(fd >= 0);
+ buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ if (buf == MAP_FAILED) {
+ return NULL;
+ }
+
+ return buf;
+}
+#else
+#error "Mirror JIT unimplemented for this platform."
+#endif /* CONFIG_IOS */
+#endif /* CONFIG_MIRROR_JIT */
+
+static inline void code_gen_alloc(size_t tb_size, bool mirror_jit)
{
tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size);
tcg_ctx->code_gen_buffer = alloc_code_gen_buffer();
@@ -1126,6 +1167,17 @@ static inline void code_gen_alloc(size_t tb_size)
fprintf(stderr, "Could not allocate dynamic translator buffer\n");
exit(1);
}
+#if defined(CONFIG_MIRROR_JIT)
+ void *mirror;
+
+ /* For platforms that need a mirror mapping for code execution */
+ mirror = alloc_jit_rw_mirror();
+ if (mirror == NULL) {
+ fprintf(stderr, "Could not remap code buffer mirror\n");
+ exit(1);
+ }
+ tcg_ctx->code_rw_mirror_diff = mirror - tcg_ctx->code_gen_buffer;
+#endif /* CONFIG_MIRROR_JIT */
}
static bool tb_cmp(const void *ap, const void *bp)
@@ -1721,6 +1773,9 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
cpu_loop_exit(cpu);
}
+#if defined(CONFIG_MIRROR_JIT)
+ tb->code_rw_mirror_diff = tcg_ctx->code_rw_mirror_diff;
+#endif
gen_code_buf = tcg_ctx->code_gen_ptr;
tb->tc.ptr = gen_code_buf;
tb->pc = pc;
When configured with --enable-debug-tcg on Linux, we enable JIT mirror map which separates the JIT allocation into a RX-only address and a RW-only address. The backing is a memfd_create() object. This allows us to catch bugs where we unintentionally write to the JIT region. However, the main use case will be for other platforms that require strict W^X separation to run. Signed-off-by: Joelle van Dyne <j@getutm.app> --- configure | 13 ++++++++ meson.build | 1 + include/exec/exec-all.h | 8 +++++ include/tcg/tcg.h | 18 ++++++++++ accel/tcg/cpu-exec.c | 7 +++- accel/tcg/translate-all.c | 69 +++++++++++++++++++++++++++++++++++---- 6 files changed, 108 insertions(+), 8 deletions(-)