Diagnose native MSVC ABI regression

.github/workflows/build.yaml

@@ -81,6 +81,7 @@ jobs:
             if [[ $TARGET == x86_64-pc-windows-msvc ]]; then
               command -v cl
               cl 2>&1 || true
+              echo "CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER=$(cygpath -w "${VCToolsInstallDir}bin/Hostx64/x64/link.exe")" >> "$GITHUB_ENV"
               printf '%s\n' "$VCToolsVersion" "$WindowsSDKVersion" "$VCToolsInstallDir" "$WindowsSdkDir"
             else
               gcc --version
@@ -124,6 +125,7 @@ jobs:
           fi
           python scripts/test-ci.py
       - name: Native ABI regression
+        id: native_abi
         env:
           TARGET: ${{ matrix.target }}
         run: |
@@ -136,6 +138,17 @@ jobs:
           cmake -S crates/core/native -B ".cache/native-tests/$TARGET" -DCMAKE_BUILD_TYPE=Release "${options[@]}"
           cmake --build ".cache/native-tests/$TARGET" --config Release --parallel 2 --target v8_killer_abi_test
           ".cache/native-tests/$TARGET/$executable"
+      - name: Upload native ABI diagnostics
+        if: failure() && steps.native_abi.outcome == 'failure'
+        uses: actions/upload-artifact@v4
+        with:
+          name: native-abi-${{ matrix.target }}
+          path: |
+            .cache/native-tests/${{ matrix.target }}/abi_test.asm
+            .cache/native-tests/${{ matrix.target }}/Release/v8_killer_abi_test.exe
+            .cache/native-tests/${{ matrix.target }}/v8_killer_abi_test
+          retention-days: 7
+          if-no-files-found: ignore
       - name: Build, test and lint
         env:
           TARGET: ${{ matrix.target }}

crates/core/native/CMakeLists.txt

@@ -30,7 +30,11 @@ install(TARGETS v8_killer_native dobby ARCHIVE DESTINATION lib)
 install(FILES ${dobby_SOURCE_DIR}/LICENSE DESTINATION share/licenses/dobby)
 
 add_executable(v8_killer_abi_test EXCLUDE_FROM_ALL abi_test.cc)
+target_include_directories(v8_killer_abi_test PRIVATE ${dobby_SOURCE_DIR}/source ${dobby_SOURCE_DIR}/source/include ${dobby_SOURCE_DIR}/external/logging)
 target_link_libraries(v8_killer_abi_test PRIVATE v8_killer_native dobby ${CMAKE_DL_LIBS})
+if(MSVC)
+  target_compile_options(v8_killer_abi_test PRIVATE /FAs "/Fa${CMAKE_CURRENT_BINARY_DIR}/abi_test.asm")
+endif()
 if(WIN32)
   target_link_libraries(v8_killer_abi_test PRIVATE psapi)
 endif()

crates/core/native/abi_test.cc

@@ -2,6 +2,9 @@
 #include <cstdlib>
 #include <cstdio>
 #include <cinttypes>
+extern "C" {
+#include "InstructionRelocation/x86/x86_insn_decode/x86_insn_decode.h"
+}
 
 extern "C" int v8_killer_instrument(void *);
 extern "C" int v8_killer_instrument_module(void *);
@@ -52,13 +55,39 @@ static NOINLINE MaybeLocal compile_module(void *isolate, uintptr_t *source,
   return MaybeLocal((uintptr_t)isolate + *source + options + reason);
 }
 
+static void print_entry(const char *name, const void *address) {
+  auto bytes = static_cast<const unsigned char *>(address);
+  std::printf("%s entry:", name);
+  for (size_t i = 0; i < 32; ++i) std::printf(" %02x", static_cast<unsigned int>(bytes[i]));
+  std::putchar('\n');
+}
+
 int main() {
   std::setvbuf(stdout, nullptr, _IONBF, 0);
+  uint8_t instructions[16] = {0x48, 0x63, 0x44, 0x24, 0x48, 0x4c, 0x63, 0x54, 0x24, 0x40};
+  x86_options_t config = {};
+  config.mode = 64;
+  for (size_t offset = 0; offset < 10; offset += 5) {
+    x86_insn_decode_t decoded = {};
+    x86_insn_decode(&decoded, instructions + offset, &config);
+    std::printf("decoder offset=%zu length=%u rex=%02x opcode=%02x operands=%c%c,%c%c flags=%u\n",
+                offset, unsigned(decoded.length), unsigned(decoded.rex), unsigned(decoded.primary_opcode),
+                decoded.insn_spec.operands[0].code, decoded.insn_spec.operands[0].type,
+                decoded.insn_spec.operands[1].code, decoded.insn_spec.operands[1].type, unsigned(decoded.flags));
+    if (decoded.length != 5) return 8;
+  }
   uintptr_t source = 10;
   auto volatile internal = compile_internal;
   auto volatile public_compile = compile;
+  auto baseline = public_compile(Local{(void *)1}, &source, 2, (Local *)3, 4, (Local *)5, 6, 7);
+  std::printf("CompileFunction baseline: source=%" PRIuPTR " expected=10 result=%" PRIuPTR " expected=38\n",
+              source, baseline.value);
+  if (source != 10 || baseline.value != 38) return 7;
   std::printf("CompileFunctionInternal=%p CompileFunction=%p CompileModule=%p source=%p\n",
               (void *)compile_internal, (void *)compile, (void *)compile_module, (void *)&source);
+  print_entry("CompileFunctionInternal original", (void *)compile_internal);
+  print_entry("CompileFunction original", (void *)compile);
+  print_entry("CompileModule original", (void *)compile_module);
   auto status = v8_killer_instrument((void *)compile_internal);
   std::printf("CompileFunctionInternal hook status=%d\n", status);
   if (status) return 1;
@@ -69,6 +98,7 @@ int main() {
   status = v8_killer_instrument((void *)compile);
   std::printf("CompileFunction hook status=%d\n", status);
   if (status) return 3;
+  print_entry("CompileFunction patched", (void *)compile);
   source = 10;
   seen_context = nullptr;
   seen_source = nullptr;

crates/core/native/patch.cmake

@@ -18,6 +18,24 @@ if(NOT content MATCHES "PAGE_READWRITE")
 endif()
 file(WRITE "${windows}" "${content}")
 
+set(decoder "${dobby_SOURCE_DIR}/source/InstructionRelocation/x86/x86_insn_decode/x86_insn_decode.c")
+set(arena "${dobby_SOURCE_DIR}/source/MemoryAllocator/NearMemoryArena.cc")
+file(READ "${arena}" content)
+string(REPLACE "#if defined(WIN32)" "#if defined(_WIN32)" content "${content}")
+file(WRITE "${arena}" "${content}")
+
+file(READ "${decoder}" content)
+string(FIND "${content}" "#define op3_flag" macro_start)
+string(FIND "${content}" "#define op2_flag" macro_end)
+string(SUBSTRING "${content}" 0 ${macro_start} before_macro)
+string(SUBSTRING "${content}" ${macro_end} -1 after_macro)
+# MSVC drops the nested array designators in the upstream operand table initializer.
+file(WRITE "${decoder}" "${before_macro}#define op3_flag(x, f, o0, o1, o2) { #x, { { #o0 }, { #o1 }, { #o2 } }, (f) }\n${after_macro}")
+set(decoder_header "${dobby_SOURCE_DIR}/source/InstructionRelocation/x86/x86_insn_decode/x86_insn_decode.h")
+file(READ "${decoder_header}" content)
+string(REPLACE "  struct {\n    uint8_t code;\n    uint8_t type;\n  };\n  uint8_t data[2];" "  uint8_t data[2];\n  struct {\n    uint8_t code;\n    uint8_t type;\n  };" content "${content}")
+file(WRITE "${decoder_header}" "${content}")
+
 set(build_file "${dobby_SOURCE_DIR}/CMakeLists.txt")
 file(READ "${build_file}" content)
 string(REPLACE "string(TIMESTAMP TODAY \"%Y%m%d\")" "set(TODAY \"20210615-223aabced043\")" content "${content}")