allow assembler to skip comment removal to avoid removing comments twice

Since the preprocessor was introduced, which already removes comments, the
assembler does not need to remove comments anymore in the usual case. The
assembler still retains the ability to remove comments (enabled by default)
in case it is used without the preprocessor. The `remove_comments` argument
to the `assemble()` method can be used to control whether comments will be
removed during assembly or not.

Author: wnienhaus

esp32_ulp/__main__.py

@@ -10,7 +10,7 @@
 
 def src_to_binary(src):
     assembler = Assembler()
-    assembler.assemble(src)
+    assembler.assemble(src, remove_comments=False)  # comments already removed by preprocessor
     garbage_collect('before symbols export')
     addrs_syms = assembler.symbols.export()
     for addr, sym in addrs_syms:

esp32_ulp/assemble.py

@@ -3,7 +3,7 @@
 """
 
 from . import opcodes
-from .nocomment import remove_comments
+from .nocomment import remove_comments as do_remove_comments
 from .util import garbage_collect
 
 TEXT, DATA, BSS = 'text', 'data', 'bss'
@@ -283,8 +283,8 @@ def assembler_pass(self, lines):
                 raise ValueError('Unknown opcode or directive: %s' % opcode)
         self.finalize_sections()
 
-    def assemble(self, text):
-        lines = remove_comments(text)
+    def assemble(self, text, remove_comments=True):
+        lines = do_remove_comments(text) if remove_comments else text.splitlines()
         self.init(1)  # pass 1 is only to get the symbol table right
         self.assembler_pass(lines)
         self.symbols.set_bases(self.compute_bases())

tests/assemble.py

@@ -157,6 +157,24 @@ def test_assemble_evalulate_expressions():
     assert a.symbols.get_sym('entry') == (REL, TEXT, 0)
 
 
+def test_assemble_optional_comment_removal():
+    line = " move r1, 123  # comment"
+
+    a = Assembler()
+
+    # first assemble as normal (comments will be removed by default)
+    a.assemble(line)
+
+    # now assemble with comment removal disabled
+    try:
+        a.assemble(line, remove_comments=False)
+    except ValueError as e:
+        raised = True
+    else:
+        raised = False
+    assert raised
+
+
 def test_symbols():
     st = SymbolTable({}, {}, {})
     for entry in [
@@ -218,4 +236,5 @@ def test_symbols():
 test_assemble_global()
 test_assemble_uppercase_opcode()
 test_assemble_evalulate_expressions()
+test_assemble_optional_comment_removal()
 test_symbols()