improve line parsing

So far line parsing made many assumptions on the format of lines.
This commit makes line parsing more flexible and tolerable to
different formats.

The change is in how labels are identified. Now any symbol that
is first on the line (excluding whitespace) and that is followed
directly with a colon (:), will be identified as a label.

Labels allow all characters that are valid for symbol names,
including ._$.

This commit contributes to being able to eventually assemble the
esp32ulp_all.s test from binutils-esp32ulp. It addresses this line:
[esp32ulp_all.s:2](https://github.com/espressif/binutils-esp32ulp/blob/249ec34cc2c9574a86f3f86bbb175a863f988bcf/gas/testsuite/gas/esp32ulp/esp32/esp32ulp_all.s#L2) (no space between colon and next character)
and also this line: [esp32ulp_globals.s:92](https://github.com/espressif/binutils-esp32ulp/blob/249ec34cc2c9574a86f3f86bbb175a863f988bcf/gas/testsuite/gas/esp32ulp/esp32/esp32ulp_globals.s#L92) (label indented).

Author: wnienhaus

esp32_ulp/assemble.py

@@ -108,17 +108,21 @@ def parse_line(self, line):
         """
         if not line:
             return
-        has_label = line[0] not in '\t .'
+        has_label = ':' in line
         if has_label:
-            label_line = line.split(None, 1)
+            orig_line = line.strip()
+            label_line = orig_line.split(':', 1)
             if len(label_line) == 2:
                 label, line = label_line
             else:  # 1
                 label, line = label_line[0], None
-            label = label.rstrip(':')
+
+            if label.strip('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_$.'):  # if any chars remain
+                # if label contains other chars than allowed, it's not a label
+                label, line = None, orig_line
         else:
             label, line = None, line.lstrip()
-        if line is None:
+        if not line:
             opcode, args = None, ()
         else:
             opcode_args = line.split(None, 1)

tests/assemble.py

@@ -53,6 +53,30 @@ def test_parse_line():
     assert a.parse_line(next(lines)) == (None, '.data', ())  # test left-aligned directive is not treated as label
 
 
+def test_parse_labels_correctly():
+    """
+    description of what defines a label
+    https://sourceware.org/binutils/docs/as/Statements.html
+    https://sourceware.org/binutils/docs/as/Labels.html
+    """
+    a = Assembler()
+    assert a.parse_line('label: .set const, 42') == ('label', '.set', ('const', '42',))
+    assert a.parse_line('label:.set const, 42') == ('label', '.set', ('const', '42',))
+    assert a.parse_line('label:') == ('label', None, ())
+    assert a.parse_line('    label:') == ('label', None, ())
+    assert a.parse_line('    label:  ') == ('label', None, ())
+    assert a.parse_line('nop  ') == (None, 'nop', ())
+    assert a.parse_line('.set c, 1  ') == (None, '.set', ('c', '1',))
+    assert a.parse_line('invalid : nop') == (None, 'invalid', (': nop',))  # no whitespace between label and colon
+    assert a.parse_line('.string "hello world"') == (None, '.string', ('"hello world"',))
+    assert a.parse_line('.string "hello : world"') == (None, '.string', ('"hello : world"',))  # colon in string
+    assert a.parse_line('label::') == ('label', ':', ())
+    assert a.parse_line('label: :') == ('label', ':', ())
+    assert a.parse_line('a_label:') == ('a_label', None, ())
+    assert a.parse_line('$label:') == ('$label', None, ())
+    assert a.parse_line('.label:') == ('.label', None, ())
+
+
 def test_parse():
     a = Assembler()
     lines = remove_comments(src)
@@ -260,6 +284,7 @@ def test_support_multiple_statements_per_line():
 
 
 test_parse_line()
+test_parse_labels_correctly()
 test_parse()
 test_assemble()
 test_assemble_bss()