Track source position information in Streams

Prior to this change, the only positional variables which were tracked
when moving through the stream was the current index in the stream.

We would like to start reporting richer positional information from the
parser, specifically the position (in terms of row/column) in the source
object.

This change allows this by updating the Stream class to additionally
track its current (zero-indexed) row and column while moving through the
stream. We will persist this information into the syntax tree in the
following commits.

Author: leamingrad

fluent.syntax/fluent/syntax/stream.py

@@ -9,7 +9,11 @@ class ParserStream:
     def __init__(self, string: str):
         self.string = string
         self.index = 0
+        self.row_index = 0
+        self.column_index = 0
         self.peek_offset = 0
+        self._peek_row_offset = 0
+        self._peek_column_offset = 0
 
     def get(self, offset: int) -> Union[str, None]:
         try:
@@ -37,9 +41,19 @@ def current_peek(self) -> Union[str, None]:
 
     def next(self) -> Union[str, None]:
         self.peek_offset = 0
+        self._peek_row_offset = 0
+        self._peek_column_offset = 0
+
         # Skip over CRLF as if it was a single character.
         if self.get(self.index) == "\r" and self.get(self.index + 1) == "\n":
             self.index += 1
+        # If we have reached a newline reset the position
+        if self.get(self.index) == "\n":
+            self.row_index += 1
+            self.column_index = 0
+        else:
+            self.column_index += 1
+
         self.index += 1
         return self.get(self.index)
 
@@ -50,6 +64,13 @@ def peek(self) -> Union[str, None]:
             and self.get(self.index + self.peek_offset + 1) == "\n"
         ):
             self.peek_offset += 1
+
+        if self.get(self.index + self.peek_offset) == "\n":
+            self._peek_row_offset += 1
+            self._peek_column_offset = 0
+        else:
+            self._peek_column_offset += 1
+
         self.peek_offset += 1
         return self.get(self.index + self.peek_offset)
 
@@ -58,7 +79,17 @@ def reset_peek(self, offset: int = 0) -> None:
 
     def skip_to_peek(self) -> None:
         self.index += self.peek_offset
+        self.row_index += self._peek_row_offset
+        if self._peek_row_offset:
+            # There have been newlines during the peek, so the column offset is the column index
+            # since the last newline
+            self.column_index = self._peek_column_offset
+        else:
+            self.column_index += self._peek_column_offset
+
         self.peek_offset = 0
+        self._peek_row_offset = 0
+        self._peek_column_offset = 0
 
 
 EOL = "\n"

fluent.syntax/tests/syntax/test_stream.py

@@ -106,20 +106,26 @@ def test_skip_to_peek(self):
         self.assertEqual("c", ps.current_peek)
         self.assertEqual(0, ps.peek_offset)
         self.assertEqual(2, ps.index)
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(2, ps.column_index)
 
         ps.peek()
 
         self.assertEqual("c", ps.current_char)
         self.assertEqual("d", ps.current_peek)
         self.assertEqual(1, ps.peek_offset)
         self.assertEqual(2, ps.index)
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(2, ps.column_index)
 
         ps.next()
 
         self.assertEqual("d", ps.current_char)
         self.assertEqual("d", ps.current_peek)
         self.assertEqual(0, ps.peek_offset)
         self.assertEqual(3, ps.index)
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(3, ps.column_index)
 
     def test_reset_peek(self):
         ps = ParserStream("abcd")
@@ -159,3 +165,113 @@ def test_reset_peek(self):
 
         self.assertEqual("d", ps.peek())
         self.assertEqual(None, ps.peek())
+
+    def test_next_with_newline(self):
+        ps = ParserStream("ab\ncd\r\nef")
+
+        self.assertEqual("a", ps.current_char)
+        self.assertEqual(0, ps.index)
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+        self.assertEqual("b", ps.next())
+        self.assertEqual("b", ps.current_char)
+        self.assertEqual(1, ps.index)
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(1, ps.column_index)
+
+
+        self.assertEqual("\n", ps.next())
+        self.assertEqual("\n", ps.current_char)
+        self.assertEqual(2, ps.index)
+        self.assertEqual(0, ps.row_index)
+        self.assertEqual(2, ps.column_index)
+
+        self.assertEqual("c", ps.next())
+        self.assertEqual("c", ps.current_char)
+        self.assertEqual(3, ps.index)
+        self.assertEqual(1, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+        self.assertEqual("d", ps.next())
+        self.assertEqual("d", ps.current_char)
+        self.assertEqual(4, ps.index)
+        self.assertEqual(1, ps.row_index)
+        self.assertEqual(1, ps.column_index)
+
+        self.assertEqual("\r", ps.next())
+        self.assertEqual("\n", ps.current_char)
+        self.assertEqual(5, ps.index)
+        self.assertEqual(1, ps.row_index)
+        self.assertEqual(2, ps.column_index)
+
+        self.assertEqual("e", ps.next())
+        self.assertEqual("e", ps.current_char)
+        self.assertEqual(7, ps.index)
+        self.assertEqual(2, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+        self.assertEqual("f", ps.next())
+        self.assertEqual("f", ps.current_char)
+        self.assertEqual(8, ps.index)
+        self.assertEqual(2, ps.row_index)
+        self.assertEqual(1, ps.column_index)
+
+        self.assertEqual(None, ps.next())
+        self.assertEqual(None, ps.current_char)
+        self.assertEqual(9, ps.index)
+        self.assertEqual(2, ps.row_index)
+        self.assertEqual(2, ps.column_index)
+
+    def test_skip_to_peek_with_newlines(self):
+        ps = ParserStream("ab\ncd\r\nef")
+
+        ps.peek()
+        ps.peek()
+        ps.peek()
+
+        ps.skip_to_peek()
+
+        self.assertEqual("c", ps.current_char)
+        self.assertEqual("c", ps.current_peek)
+        self.assertEqual(0, ps.peek_offset)
+        self.assertEqual(3, ps.index)
+        self.assertEqual(1, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+        ps.peek()
+
+        self.assertEqual("c", ps.current_char)
+        self.assertEqual("d", ps.current_peek)
+        self.assertEqual(1, ps.peek_offset)
+        self.assertEqual(3, ps.index)
+        self.assertEqual(1, ps.row_index)
+        self.assertEqual(0, ps.column_index)
+
+        ps.next()
+
+        self.assertEqual("d", ps.current_char)
+        self.assertEqual("d", ps.current_peek)
+        self.assertEqual(0, ps.peek_offset)
+        self.assertEqual(4, ps.index)
+        self.assertEqual(1, ps.row_index)
+        self.assertEqual(1, ps.column_index)
+
+        ps.peek()
+        ps.peek()
+
+        self.assertEqual("d", ps.current_char)
+        self.assertEqual("e", ps.current_peek)
+        self.assertEqual(3, ps.peek_offset)
+        self.assertEqual(4, ps.index)
+        self.assertEqual(1, ps.row_index)
+        self.assertEqual(1, ps.column_index)
+
+        ps.skip_to_peek()
+
+        self.assertEqual("e", ps.current_char)
+        self.assertEqual("e", ps.current_peek)
+        self.assertEqual(0, ps.peek_offset)
+        self.assertEqual(7, ps.index)
+        self.assertEqual(2, ps.row_index)
+        self.assertEqual(0, ps.column_index)