wasm: preserve Memory64 DWARF through Binaryen rewrites

test/unit/input/dwarf-memory64.c

@@ -0,0 +1,12 @@
+__attribute__((always_inline)) static inline long add_seven(long input) {
+  long local = input + 7;
+  return local;
+}
+
+__attribute__((noinline)) long debug_probe(long input) {
+  long before = add_seven(input);
+  if (input > 0) {
+    before += add_seven(input + 1);
+  }
+  return before;
+}

test/unit/test_dwarf.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import subprocess
 import tempfile
 
@@ -7,7 +8,179 @@
 from . import utils
 
 
+def read_uleb(data, offset):
+    value = 0
+    shift = 0
+    while True:
+        byte = data[offset]
+        offset += 1
+        value |= (byte & 0x7f) << shift
+        if byte < 0x80:
+            return value, offset
+        shift += 7
+
+
+def write_uleb(value):
+    data = bytearray()
+    while value >= 0x80:
+        data.append((value & 0x7f) | 0x80)
+        value >>= 7
+    data.append(value)
+    return bytes(data)
+
+
+def custom_section(data, name):
+    offset = 8
+    while offset < len(data):
+        kind = data[offset]
+        size, payload = read_uleb(data, offset + 1)
+        end = payload + size
+        if kind == 0:
+            name_size, contents = read_uleb(data, payload)
+            section_name = data[contents:contents + name_size].decode()
+            if section_name == name:
+                return data[contents + name_size:end]
+        offset = end
+    raise ValueError(name)
+
+
+def replace_custom_section(data, name, replacement):
+    offset = 8
+    while offset < len(data):
+        kind = data[offset]
+        size, payload = read_uleb(data, offset + 1)
+        end = payload + size
+        if kind == 0:
+            name_size, contents = read_uleb(data, payload)
+            section_name = data[contents:contents + name_size].decode()
+            if section_name == name:
+                name_bytes = data[contents:contents + name_size]
+                new_payload = write_uleb(name_size) + name_bytes + replacement
+                return (data[:offset] + b'\0' +
+                        write_uleb(len(new_payload)) + new_payload + data[end:])
+        offset = end
+    raise ValueError(name)
+
+
+def append_custom_section(data, name, contents):
+    name_bytes = name.encode()
+    payload = write_uleb(len(name_bytes)) + name_bytes + contents
+    return data + b'\0' + write_uleb(len(payload)) + payload
+
+
 class DWARFTest(utils.BinaryenTestCase):
+    def test_memory64_address_width(self):
+        # Regenerate the checked-in input from dwarf-memory64.c with:
+        # clang -target wasm64-unknown-unknown -O1 -g -gdwarf-4 \
+        #   -fdebug-compilation-dir=/binaryen -c dwarf-memory64.c -o input.o
+        # wasm-ld -mwasm64 --no-entry --export=debug_probe input.o -o memory64.wasm
+        source = self.input_path(os.path.join('dwarf', 'memory64.wasm'))
+        dwarfdump = shutil.which('llvm-dwarfdump')
+        with tempfile.TemporaryDirectory() as temp_dir:
+            for name, args in [('roundtrip', ['--roundtrip']),
+                               ('asyncify', ['--asyncify'])]:
+                output = os.path.join(temp_dir, name + '.wasm')
+                shared.run_process(shared.WASM_OPT +
+                                   [source, '-g', *args, '-o', output])
+                dump = shared.run_process(shared.WASM_OPT +
+                                          [output, '--dwarfdump'],
+                                          capture_output=True).stdout
+                self.assertIn('debug_probe', dump)
+                if dwarfdump:
+                    verify = subprocess.run([dwarfdump, '--verify', output],
+                                            capture_output=True, text=True)
+                    diagnostics = verify.stdout + verify.stderr
+                    self.assertEqual(verify.returncode, 0, diagnostics)
+                    self.assertNotIn('mismatching address size', diagnostics)
+                    self.assertIn('No errors.', diagnostics)
+
+    def test_memory64_dwarf64_line_discriminator(self):
+        source = self.input_path(os.path.join('dwarf', 'memory64.wasm'))
+        with open(source, 'rb') as f:
+            wasm = f.read()
+        line = custom_section(wasm, '.debug_line')
+        prologue_length = int.from_bytes(line[6:10], 'little')
+        opcodes = 10 + prologue_length
+        set_address = line.index(b'\x00\x09\x02', opcodes)
+        first_row = set_address + 11  # opcode, length, subopcode, 8-byte address
+        discriminator = b'\x00\x03\x04\xac\x02'  # DW_LNE_set_discriminator 300
+        line = line[:first_row] + discriminator + line[first_row:]
+        # Convert the line table to DWARF64, independently of the CU's
+        # address size and the DWARF32 format used by .debug_info.
+        body_length = len(line)  # four more prologue bytes replace four prefix bytes
+        line = (b'\xff' * 4 + body_length.to_bytes(8, 'little') +
+                line[4:6] + prologue_length.to_bytes(8, 'little') + line[10:])
+        wasm = replace_custom_section(wasm, '.debug_line', line)
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            input_file = os.path.join(temp_dir, 'input.wasm')
+            output_file = os.path.join(temp_dir, 'output.wasm')
+            with open(input_file, 'wb') as f:
+                f.write(wasm)
+            shared.run_process(shared.WASM_OPT +
+                               [input_file, '--roundtrip', '-g', '-o', output_file])
+            with open(output_file, 'rb') as f:
+                output = f.read()
+            output_line = custom_section(output, '.debug_line')
+            self.assertEqual(output_line[:4], b'\xff' * 4)
+            self.assertEqual(int.from_bytes(output_line[4:12], 'little'),
+                             len(output_line) - 12)
+            self.assertIn(discriminator, output_line)
+            dwarfdump = shutil.which('llvm-dwarfdump')
+            if dwarfdump:
+                verify = subprocess.run([dwarfdump, '--verify', output_file],
+                                        capture_output=True, text=True)
+                self.assertEqual(verify.returncode, 0,
+                                 verify.stdout + verify.stderr)
+                lines = subprocess.run([dwarfdump, '--debug-line', output_file],
+                                       capture_output=True, text=True)
+                self.assertIn('DWARF64', lines.stdout)
+                self.assertIn('300', lines.stdout)
+
+    def test_memory64_loc_value_with_32_bit_all_ones(self):
+        source = self.input_path(os.path.join('dwarf', 'memory64.wasm'))
+        with open(source, 'rb') as f:
+            wasm = f.read()
+        loc = custom_section(wasm, '.debug_loc')
+        # A 64-bit value with only its low 32 bits set is a normal location,
+        # not a base-address-selection marker or list terminator.
+        loc = ((0xffffffff).to_bytes(8, 'little') +
+               (0x100000000).to_bytes(8, 'little') + loc[16:])
+        wasm = replace_custom_section(wasm, '.debug_loc', loc)
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            input_file = os.path.join(temp_dir, 'input.wasm')
+            output_file = os.path.join(temp_dir, 'output.wasm')
+            with open(input_file, 'wb') as f:
+                f.write(wasm)
+            shared.run_process(shared.WASM_OPT +
+                               [input_file, '--roundtrip', '-g', '-o', output_file])
+            with open(output_file, 'rb') as f:
+                output_loc = custom_section(f.read(), '.debug_loc')
+            self.assertEqual(len(output_loc), len(loc))
+            self.assertEqual(output_loc[:16],
+                             (1).to_bytes(8, 'little') * 2)
+
+    def test_memory64_range_outside_binary_offset(self):
+        source = self.input_path(os.path.join('dwarf', 'memory64.wasm'))
+        with open(source, 'rb') as f:
+            wasm = f.read()
+        ranges = ((0x100000003).to_bytes(8, 'little') +
+                  (0x100000004).to_bytes(8, 'little') + b'\0' * 16)
+        wasm = append_custom_section(wasm, '.debug_ranges', ranges)
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            input_file = os.path.join(temp_dir, 'input.wasm')
+            output_file = os.path.join(temp_dir, 'output.wasm')
+            with open(input_file, 'wb') as f:
+                f.write(wasm)
+            shared.run_process(shared.WASM_OPT +
+                               [input_file, '--roundtrip', '-g', '-o', output_file])
+            with open(output_file, 'rb') as f:
+                output_ranges = custom_section(f.read(), '.debug_ranges')
+            self.assertEqual(output_ranges[:16],
+                             (1).to_bytes(8, 'little') * 2)
+
     def test_tombstone_roundtrip(self):
         def custom_section(name, contents):
             name = name.encode()

third_party/llvm-project/DWARFEmitter.cpp

@@ -124,24 +124,25 @@ void DWARFYAML::EmitDebugRanges(raw_ostream &OS, const DWARFYAML::Data &DI) {
   // format is totally trivial, consisting just of pairs of address
   // sized addresses describing the ranges." and apparently it ends
   // with a null termination of a pair of zeros
+  const auto AddrSize = DI.CompileUnits.empty() ? 4 : DI.CompileUnits[0].AddrSize;
   for (auto Range : DI.Ranges) {
-    writeInteger((uint32_t)Range.Start, OS, DI.IsLittleEndian);
-    writeInteger((uint32_t)Range.End, OS, DI.IsLittleEndian);
+    writeVariableSizedInteger(Range.Start, AddrSize, OS, DI.IsLittleEndian);
+    writeVariableSizedInteger(Range.End, AddrSize, OS, DI.IsLittleEndian);
   }
 }
 
 // XXX BINARYEN
 void DWARFYAML::EmitDebugLoc(raw_ostream &OS, const DWARFYAML::Data &DI) {
+  const auto AddrSize = DI.CompileUnits.empty() ? 4 : DI.CompileUnits[0].AddrSize;
   for (auto Loc : DI.Locs) {
-    auto AddrSize = DI.CompileUnits[0].AddrSize;  // XXX BINARYEN
-    // FIXME: Loc.Start etc should probably not be 32-bit.
-    writeVariableSizedInteger((uint64_t)(int32_t)Loc.Start, AddrSize, OS, DI.IsLittleEndian);
-    writeVariableSizedInteger((uint64_t)(int32_t)Loc.End, AddrSize, OS, DI.IsLittleEndian);
+    writeVariableSizedInteger(Loc.Start, AddrSize, OS, DI.IsLittleEndian);
+    writeVariableSizedInteger(Loc.End, AddrSize, OS, DI.IsLittleEndian);
     if (Loc.Start == 0 && Loc.End == 0) {
       // End of a list.
       continue;
     }
-    if (Loc.Start != -1) {
+    const uint64_t baseMarker = AddrSize == 8 ? uint64_t(-1) : uint32_t(-1);
+    if (Loc.Start != baseMarker) {
       writeInteger((uint16_t)Loc.Location.size(), OS, DI.IsLittleEndian);
       for (auto x : Loc.Location) {
         writeInteger((uint8_t)x, OS, DI.IsLittleEndian);
@@ -306,10 +307,12 @@ static void EmitDebugLineInternal(raw_ostream &RealOS,
         writeInteger((uint8_t)Op.SubOpcode, OS, DI.IsLittleEndian);
         switch (Op.SubOpcode) {
         case dwarf::DW_LNE_set_address:
-        case dwarf::DW_LNE_set_discriminator:
           writeVariableSizedInteger(Op.Data, DI.CompileUnits[0].AddrSize, OS,
                                     DI.IsLittleEndian);
           break;
+        case dwarf::DW_LNE_set_discriminator:
+          encodeULEB128(Op.Data, OS);
+          break;
         case dwarf::DW_LNE_define_file:
           EmitFileEntry(OS, Op.FileEntry);
           break;
@@ -352,15 +355,19 @@ static void EmitDebugLineInternal(raw_ostream &RealOS,
       }
     }
     // XXX BINARYEN Write to the actual stream, with the proper size.
-    // We assume for now that the length fits in 32 bits.
     size_t Size = OS.str().size();
-    if (Size >= UINT32_MAX) {
+    if (!LineTable.Length.isDWARF64() && Size >= UINT32_MAX) {
       llvm_unreachable("Table is too big");
     }
     if (computedLengths) {
       computedLengths->push_back(Size);
     }
-    writeInteger((uint32_t)Size, RealOS, DI.IsLittleEndian);
+    if (LineTable.Length.isDWARF64()) {
+      writeInteger(UINT32_MAX, RealOS, DI.IsLittleEndian);
+      writeInteger((uint64_t)Size, RealOS, DI.IsLittleEndian);
+    } else {
+      writeInteger((uint32_t)Size, RealOS, DI.IsLittleEndian);
+    }
     RealOS << OS.str();
   }
 }

third_party/llvm-project/dwarf2yaml.cpp

@@ -90,7 +90,11 @@ void dumpDebugARanges(DWARFContext &DCtx, DWARFYAML::Data &Y) {
 }
 
 void dumpDebugRanges(DWARFContext &DCtx, DWARFYAML::Data &Y) { // XXX BINARYEN
-  uint8_t savedAddressByteSize = 4;
+  auto CUS = DCtx.normal_units();
+  if (CUS.empty()) {
+    return;
+  }
+  uint8_t savedAddressByteSize = CUS.begin()->get()->getAddressByteSize();
   DWARFDataExtractor rangesData(DCtx.getDWARFObj(), DCtx.getDWARFObj().getRangesSection(),
                                 DCtx.isLittleEndian(), savedAddressByteSize);
   uint64_t offset = 0;
@@ -368,7 +372,7 @@ void dumpDebugLines(DWARFContext &DCtx, DWARFYAML::Data &Y) {
       }
 
       const uint64_t LineEnd =
-          LineTableLength + *StmtOffset + SizeOfPrologueLength;
+          LineTableLength + *StmtOffset + (DebugLines.Length.isDWARF64() ? 12 : 4);
       while (Offset < LineEnd) {
         DWARFYAML::LineTableOpcode NewOp = {};
         NewOp.Opcode = (dwarf::LineNumberOps)LineData.getU8(&Offset);
@@ -379,9 +383,11 @@ void dumpDebugLines(DWARFContext &DCtx, DWARFYAML::Data &Y) {
               (dwarf::LineNumberExtendedOps)LineData.getU8(&Offset);
           switch (NewOp.SubOpcode) {
           case dwarf::DW_LNE_set_address:
-          case dwarf::DW_LNE_set_discriminator:
             NewOp.Data = LineData.getAddress(&Offset);
             break;
+          case dwarf::DW_LNE_set_discriminator:
+            NewOp.Data = LineData.getULEB128(&Offset);
+            break;
           case dwarf::DW_LNE_define_file:
             dumpFileEntry(LineData, Offset, NewOp.FileEntry);
             break;

third_party/llvm-project/include/llvm/ObjectYAML/DWARFYAML.h

@@ -89,8 +89,8 @@ struct Range {
 };
 
 struct Loc {
-  uint32_t Start;
-  uint32_t End;
+  uint64_t Start;
+  uint64_t End;
   std::vector<uint8_t> Location;
   uint64_t CompileUnitOffset;
 };