Extend state stacks indefinitely

include/prism/internal/arena.h

@@ -108,28 +108,38 @@ pm_arena_memdup(pm_arena_t *arena, const void *src, size_t size, size_t alignmen
 /*
  * Resize an allocation from the arena from old_size to new_size bytes and
  * return where it now lives. The sizes are in the same order as xrealloc_sized.
+ * An old_size of 0 means there is no allocation yet, so ptr may be NULL and a
+ * new allocation is returned.
+ *
  * The most recent allocation in the current block ends exactly where the
  * block's used bytes do, so it is resized in place when the block has room. Any
  * other allocation stays where it is when it shrinks, and is copied into a new
  * allocation when it grows, leaving the old one behind in the arena.
  */
 static PRISM_INLINE void *
 pm_arena_realloc(pm_arena_t *arena, void *ptr, size_t new_size, size_t old_size, size_t alignment) {
-    pm_arena_block_t *block = arena->current;
+    void *result;
+
+    if (old_size == 0) {
+        result = pm_arena_alloc(arena, new_size, alignment);
+    } else {
+        pm_arena_block_t *block = arena->current;
 
-    if (block != NULL && (char *) ptr + old_size == block->data + block->used) {
-        size_t start = block->used - old_size;
+        if (block != NULL && (char *) ptr + old_size == block->data + block->used) {
+            size_t start = block->used - old_size;
 
-        if (new_size <= block->capacity - start) {
-            block->used = start + new_size;
-            return ptr;
+            if (new_size <= block->capacity - start) {
+                block->used = start + new_size;
+                return ptr;
+            }
         }
-    }
 
-    if (new_size <= old_size) return ptr;
+        if (new_size <= old_size) return ptr;
+
+        result = pm_arena_alloc(arena, new_size, alignment);
+        memcpy(result, ptr, old_size);
+    }
 
-    void *result = pm_arena_alloc(arena, new_size, alignment);
-    memcpy(result, ptr, old_size);
     return result;
 }
 

include/prism/internal/parser.h

@@ -577,9 +577,35 @@ typedef struct pm_scope {
 } pm_scope_t;
 
 /*
- * A struct that represents a stack of boolean values.
+ * A stack of boolean values with no fixed depth limit. The newest frames live
+ * in an inline word, so pushing and popping are usually a shift and a compare.
+ * When that word fills, it moves to the spill array and the inline word starts
+ * over empty. The spill array is only allocated once the nesting is more than
+ * 64 frames deep.
  */
-typedef uint32_t pm_state_stack_t;
+typedef struct {
+    /*
+     * The newest frames, with the top of the stack in bit 0. It holds between
+     * 1 and 64 frames while the stack is not empty, and is 0 when it is.
+     */
+    uint64_t top;
+
+    /*
+     * The number of frames on the whole stack. The inline word holds the
+     * frames above the last multiple of 64, so the spill array holds
+     * (depth - 1) / 64 words.
+     */
+    uint32_t depth;
+
+    /* The number of words the spill array has room for. */
+    uint32_t capacity;
+
+    /*
+     * Full words of 64 older frames each, oldest first, allocated from the
+     * scratch arena.
+     */
+    uint64_t *spill;
+} pm_state_stack_t;
 
 /*
  * This struct represents the overall parser. It contains a reference to the

src/prism.c

@@ -7272,33 +7272,73 @@ pm_parser_scope_pop(pm_parser_t *parser) {
 /******************************************************************************/
 
 /**
- * Pushes a value onto the stack.
+ * Moves the full inline word of the stack into the spill array, growing the
+ * array if it is out of room, and leaves the inline word empty.
+ */
+static void
+pm_state_stack_spill(pm_arena_t *arena, pm_state_stack_t *stack) {
+    uint32_t size = (stack->depth >> 6) - 1;
+
+    if (size == stack->capacity) {
+        uint32_t capacity = stack->capacity == 0 ? 2 : stack->capacity * 2;
+        stack->spill = (uint64_t *) pm_arena_realloc(arena, stack->spill, capacity * sizeof(uint64_t), stack->capacity * sizeof(uint64_t), PRISM_ALIGNOF(uint64_t));
+        stack->capacity = capacity;
+    }
+
+    stack->spill[size] = stack->top;
+    stack->top = 0;
+}
+
+/**
+ * Pushes a value onto the stack. The inline word is full when the depth is a
+ * nonzero multiple of 64.
  */
 static PRISM_INLINE void
-pm_state_stack_push(pm_state_stack_t *stack, bool value) {
-    *stack = (*stack << 1) | (value & 1);
+pm_state_stack_push(pm_arena_t *arena, pm_state_stack_t *stack, bool value) {
+    if (PRISM_UNLIKELY(stack->depth && !(stack->depth & 63))) {
+        pm_state_stack_spill(arena, stack);
+    }
+
+    stack->top = (stack->top << 1) | value;
+    stack->depth++;
 }
 
 /**
- * Pops a value off the stack.
+ * Pops a value off the stack. When that empties the inline word, the newest
+ * spilled word moves back into it, so the inline word is empty only when the
+ * whole stack is.
  */
 static PRISM_INLINE void
 pm_state_stack_pop(pm_state_stack_t *stack) {
-    *stack >>= 1;
+    /* A closing delimiter without a matching opening one is a syntax error that
+     * the parser reports and recovers from, but the lexer still pops for it, so
+     * popping an empty stack has to leave it empty. */
+    if (PRISM_UNLIKELY(stack->depth == 0)) return;
+
+    stack->top >>= 1;
+    stack->depth--;
+
+    if (PRISM_UNLIKELY(stack->depth && !(stack->depth & 63))) {
+        stack->top = stack->spill[(stack->depth >> 6) - 1];
+    }
 }
 
 /**
- * Returns the value at the top of the stack.
+ * Returns the value at the top of the stack. An empty stack reads as false,
+ * because its inline word is 0.
  */
 static PRISM_INLINE bool
 pm_state_stack_p(const pm_state_stack_t *stack) {
-    return *stack & 1;
+    return stack->top & 1;
 }
 
+/**
+ * The accepts-block stack stores the negation of each value, so that an empty
+ * stack (which reads as false) means blocks are accepted.
+ */
 static PRISM_INLINE void
 pm_accepts_block_stack_push(pm_parser_t *parser, bool value) {
-    // Use the negation of the value to prevent stack overflow.
-    pm_state_stack_push(&parser->accepts_block_stack, !value);
+    pm_state_stack_push(&parser->scratch_arena, &parser->accepts_block_stack, !value);
 }
 
 static PRISM_INLINE void
@@ -7313,7 +7353,7 @@ pm_accepts_block_stack_p(pm_parser_t *parser) {
 
 static PRISM_INLINE void
 pm_do_loop_stack_push(pm_parser_t *parser, bool value) {
-    pm_state_stack_push(&parser->do_loop_stack, value);
+    pm_state_stack_push(&parser->scratch_arena, &parser->do_loop_stack, value);
 }
 
 static PRISM_INLINE void
@@ -23167,8 +23207,8 @@ pm_parser_init(pm_arena_t *arena, pm_parser_t *parser, const uint8_t *source, si
         .enclosure_nesting = 0,
         .lambda_enclosure_nesting = -1,
         .brace_nesting = 0,
-        .do_loop_stack = 0,
-        .accepts_block_stack = 0,
+        .do_loop_stack = { 0 },
+        .accepts_block_stack = { 0 },
         .lex_modes = {
             .current = &parser->lex_modes.base,
             .base = { .mode = PM_LEX_DEFAULT },

test/prism/result/delimiter_nesting_test.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require_relative "../test_helper"
+
+module Prism
+  # The lexer tracks whether a `do` closes a loop predicate and whether a `do`
+  # can open a block on stacks with one frame per open delimiter. These tests
+  # nest delimiters around and across the 64-frame words those stacks spill
+  # into, and check that the frame beneath the delimiters is still read once
+  # they close.
+  class DelimiterNestingTest < TestCase
+    DEPTHS = [1, 31, 32, 33, 63, 64, 65, 127, 128, 129, 200, 1000]
+
+    def test_loop_predicate
+      DEPTHS.each do |depth|
+        source = "while #{"(" * depth}x#{")" * depth} do end"
+        assert_loop(WhileNode, source)
+      end
+    end
+
+    def test_mixed_delimiters
+      DEPTHS.each do |depth|
+        opening = "([" * depth
+        closing = "])" * depth
+        source = "until #{opening}x#{closing} do end"
+        assert_loop(UntilNode, source)
+      end
+    end
+
+    def test_nested_loop_predicate
+      DEPTHS.each do |depth|
+        inner = "while #{"(" * depth}y#{")" * depth} do end"
+        source = "while #{"(" * depth}#{inner}#{")" * depth} do end"
+
+        node = assert_loop(WhileNode, source)
+        assert_kind_of WhileNode, node.predicate.breadth_first_search { |child| child.is_a?(WhileNode) }
+      end
+    end
+
+    def test_block_inside_loop_predicate
+      DEPTHS.each do |depth|
+        source = "while #{"(" * depth}foo do end#{")" * depth} do end"
+
+        node = assert_loop(WhileNode, source)
+        call = node.predicate.breadth_first_search { |child| child.is_a?(CallNode) }
+        assert_kind_of BlockNode, call.block
+      end
+    end
+
+    # A closing delimiter without a matching opening one still pops both
+    # stacks, so these pop an empty stack, some of them more than 64 times in
+    # a row, and check that the source still parses with errors.
+    def test_unmatched_closers
+      sources = [")", "foo]", "}", "def foo; ); end", "while x) do end", ")" * 100, "]" * 100, "}" * 100]
+
+      sources.each do |source|
+        refute_empty Prism.parse(source).errors, source
+      end
+    end
+
+    private
+
+    def assert_loop(type, source)
+      result = Prism.parse(source)
+      assert_empty result.errors, source
+
+      node = result.value.statements.body.first
+      assert_kind_of type, node
+      node
+    end
+  end
+end