From d5ecb52a71e20f0db6dcc99e30510080a87d9af3 Mon Sep 17 00:00:00 2001 From: Cyan Ogilvie Date: Thu, 16 Apr 2026 10:20:33 -0300 Subject: [PATCH] tests: add test for x86_64 xor REX prefix bug in load() When load() in x86_64-gen.c generates a zero constant for a 64-bit register, it uses: o(0xc031 + REG_VALUE(r) * 0x900); REG_VALUE(r) masks to (r & 7), losing bit 3, and no orex() call emits the REX prefix needed for registers r8-r15. For TREG_R11 (used by gcall_or_jmp for indirect calls), this emits "xor %ebx,%ebx" (31 db) instead of "xor %r11d,%r11d" (45 31 db), clobbering the wrong register. The test compiles an indirect call through a null function pointer ((void(*)(void))0)() via libtcc, then inspects the generated machine code for the incorrect encoding. --- tests/Makefile | 7 ++ tests/libtcc_test_xor_rex.c | 128 ++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 tests/libtcc_test_xor_rex.c diff --git a/tests/Makefile b/tests/Makefile index 6e0f3bd6..4611e11c 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -13,6 +13,7 @@ TESTS = \ hello-run \ libtest \ libtest_mt \ + libtest_xor_rex \ test3 \ abitest \ asm-c-connect-test \ @@ -40,6 +41,9 @@ endif ifeq (,$(filter i386 x86_64,$(ARCH))) TESTS := $(filter-out asm-c-connect-test,$(TESTS)) endif +ifeq (,$(filter x86_64,$(ARCH))) + TESTS := $(filter-out libtest_xor_rex,$(TESTS)) +endif ifeq ($(OS),Windows_NT) # for libtcc_test to find libtcc.dll PATH := $(CURDIR)/$(TOP)$(if $(findstring ;,$(PATH)),;,:)$(PATH) endif @@ -103,6 +107,9 @@ libtcc_test$(EXESUF): libtcc_test.c libtcc_test_mt$(EXESUF): libtcc_test_mt.c $(CC) -o $@ $< $(CFLAGS) $(-LTCC) $(LIBS) +libtcc_test_xor_rex$(EXESUF): libtcc_test_xor_rex.c + $(CC) -o $@ $< $(CFLAGS) $(-LTCC) $(LIBS) + %-dir: @echo ------------ $@ ------------ $(MAKE) -k -C $* diff --git a/tests/libtcc_test_xor_rex.c b/tests/libtcc_test_xor_rex.c new file mode 100644 index 00000000..89bbb1a3 --- /dev/null +++ b/tests/libtcc_test_xor_rex.c @@ -0,0 +1,128 @@ +/* + * Test for x86_64 xor REX prefix bug in load() -- x86_64-gen.c + * + * Bug: when loading a 64-bit zero constant into registers r8-r15, + * load() emits: + * + * o(0xc031 + REG_VALUE(r) * 0x900); // xor r, r + * + * REG_VALUE(r) masks to (r & 7), losing bit 3, and no orex() call + * emits the REX prefix needed for extended registers. + * + * Result: r=TREG_R11(11) -> REG_VALUE=3 -> emits 31 db (xor ebx,ebx) + * Correct: should emit 45 31 db (REX.RB xor r11d,r11d) + * + * Fix: + * orex(0, r, r, 0x31); + * o(0xc0 + REG_VALUE(r) * 9); + * + * Trigger: an indirect call through a compile-time null function pointer, + * e.g. ((void(*)(void))0)(), causes gcall_or_jmp() to fall into the + * "indirect call" path which does load(TREG_R11, ). + * + * This test compiles such code via the libtcc API, then inspects the + * generated machine code for the incorrect encoding. + */ + +#if !defined __x86_64__ +#include +int main(void) { printf("SKIP (x86_64 only)\n"); return 0; } +#else + +#include +#include "libtcc.h" + +static void handle_error(void *opaque, const char *msg) +{ + fprintf(opaque, "%s\n", msg); +} + +/* + * Compiled via libtcc. The cast-to-null indirect call forces + * gcall_or_jmp() into its "else" branch (no VT_SYM, so the + * condition on line ~650 fails), which does: + * + * r = TREG_R11; + * load(r, vtop); // <-- buggy xor lands here + * o(0x41); o(0xff); // call/jmp *r + * o(0xd0 + REG_VALUE(r)); // r11 -> 0xd3 + * + * We never execute the function (it would crash); we only + * inspect the generated bytes. + */ +static const char test_code[] = + "void test(void) {\n" + " ((void(*)(void))0)();\n" + "}\n"; + +int main(int argc, char **argv) +{ + TCCState *s; + unsigned char *code; + int i; + int ret = 0; + + s = tcc_new(); + if (!s) { + fprintf(stderr, "tcc_new() failed\n"); + return 2; + } + tcc_set_error_func(s, stderr, handle_error); + + for (i = 1; i < argc; ++i) { + char *a = argv[i]; + if (a[0] == '-') { + if (a[1] == 'B') + tcc_set_lib_path(s, a + 2); + else if (a[1] == 'I') + tcc_add_include_path(s, a + 2); + else if (a[1] == 'L') + tcc_add_library_path(s, a + 2); + } + } + + tcc_set_output_type(s, TCC_OUTPUT_MEMORY); + if (tcc_compile_string(s, test_code) == -1) + return 2; + if (tcc_relocate(s) < 0) + return 2; + + code = (unsigned char *)tcc_get_symbol(s, "test"); + if (!code) { + fprintf(stderr, "symbol 'test' not found\n"); + return 2; + } + + /* + * Scan for the 'call *%r11' instruction: 41 ff d3 + * Then inspect the bytes immediately before it. + * + * Correct: 45 31 db 41 ff d3 (xor %r11d,%r11d ; call *%r11) + * Buggy: 31 db 41 ff d3 (xor %ebx,%ebx ; call *%r11) + */ + for (i = 3; i < 128; i++) { + if (code[i] == 0x41 && code[i+1] == 0xff && code[i+2] == 0xd3) { + if (i >= 3 && code[i-3] == 0x45 + && code[i-2] == 0x31 + && code[i-1] == 0xdb) { + printf("xor_rex: OK\n"); + } else if (i >= 2 && code[i-2] == 0x31 && code[i-1] == 0xdb) { + printf("xor_rex: FAIL - xor %%ebx,%%ebx (31 db) emitted" + " instead of xor %%r11d,%%r11d (45 31 db)\n"); + ret = 1; + } else { + printf("xor_rex: FAIL - unexpected bytes before" + " call *%%r11: %02x %02x %02x %02x\n", + code[i-4], code[i-3], code[i-2], code[i-1]); + ret = 1; + } + goto done; + } + } + printf("xor_rex: SKIP - call *%%r11 not found in generated code\n"); + +done: + tcc_delete(s); + return ret; +} +#endif