diff --git a/include/prism/internal/arena.h b/include/prism/internal/arena.h index d75b4a62f0..29625781c1 100644 --- a/include/prism/internal/arena.h +++ b/include/prism/internal/arena.h @@ -108,6 +108,9 @@ pm_arena_memdup(pm_arena_t *arena, const void *src, size_t size, size_t alignmen /* * Resize an allocation from the arena from old_size to new_size bytes and * return where it now lives. The sizes are in the same order as xrealloc_sized. + * An old_size of 0 means there is no allocation yet, so ptr may be NULL and a + * new allocation is returned. + * * The most recent allocation in the current block ends exactly where the * block's used bytes do, so it is resized in place when the block has room. Any * other allocation stays where it is when it shrinks, and is copied into a new @@ -115,21 +118,28 @@ pm_arena_memdup(pm_arena_t *arena, const void *src, size_t size, size_t alignmen */ static PRISM_INLINE void * pm_arena_realloc(pm_arena_t *arena, void *ptr, size_t new_size, size_t old_size, size_t alignment) { - pm_arena_block_t *block = arena->current; + void *result; + + if (old_size == 0) { + result = pm_arena_alloc(arena, new_size, alignment); + } else { + pm_arena_block_t *block = arena->current; - if (block != NULL && (char *) ptr + old_size == block->data + block->used) { - size_t start = block->used - old_size; + if (block != NULL && (char *) ptr + old_size == block->data + block->used) { + size_t start = block->used - old_size; - if (new_size <= block->capacity - start) { - block->used = start + new_size; - return ptr; + if (new_size <= block->capacity - start) { + block->used = start + new_size; + return ptr; + } } - } - if (new_size <= old_size) return ptr; + if (new_size <= old_size) return ptr; + + result = pm_arena_alloc(arena, new_size, alignment); + memcpy(result, ptr, old_size); + } - void *result = pm_arena_alloc(arena, new_size, alignment); - memcpy(result, ptr, old_size); return result; } diff --git a/include/prism/internal/parser.h b/include/prism/internal/parser.h index ba6fe77f6a..7f06cda91d 100644 --- a/include/prism/internal/parser.h +++ b/include/prism/internal/parser.h @@ -577,9 +577,35 @@ typedef struct pm_scope { } pm_scope_t; /* - * A struct that represents a stack of boolean values. + * A stack of boolean values with no fixed depth limit. The newest frames live + * in an inline word, so pushing and popping are usually a shift and a compare. + * When that word fills, it moves to the spill array and the inline word starts + * over empty. The spill array is only allocated once the nesting is more than + * 64 frames deep. */ -typedef uint32_t pm_state_stack_t; +typedef struct { + /* + * The newest frames, with the top of the stack in bit 0. It holds between + * 1 and 64 frames while the stack is not empty, and is 0 when it is. + */ + uint64_t top; + + /* + * The number of frames on the whole stack. The inline word holds the + * frames above the last multiple of 64, so the spill array holds + * (depth - 1) / 64 words. + */ + uint32_t depth; + + /* The number of words the spill array has room for. */ + uint32_t capacity; + + /* + * Full words of 64 older frames each, oldest first, allocated from the + * scratch arena. + */ + uint64_t *spill; +} pm_state_stack_t; /* * This struct represents the overall parser. It contains a reference to the diff --git a/src/prism.c b/src/prism.c index 1f44e7e915..d428de1741 100644 --- a/src/prism.c +++ b/src/prism.c @@ -7272,33 +7272,73 @@ pm_parser_scope_pop(pm_parser_t *parser) { /******************************************************************************/ /** - * Pushes a value onto the stack. + * Moves the full inline word of the stack into the spill array, growing the + * array if it is out of room, and leaves the inline word empty. + */ +static void +pm_state_stack_spill(pm_arena_t *arena, pm_state_stack_t *stack) { + uint32_t size = (stack->depth >> 6) - 1; + + if (size == stack->capacity) { + uint32_t capacity = stack->capacity == 0 ? 2 : stack->capacity * 2; + stack->spill = (uint64_t *) pm_arena_realloc(arena, stack->spill, capacity * sizeof(uint64_t), stack->capacity * sizeof(uint64_t), PRISM_ALIGNOF(uint64_t)); + stack->capacity = capacity; + } + + stack->spill[size] = stack->top; + stack->top = 0; +} + +/** + * Pushes a value onto the stack. The inline word is full when the depth is a + * nonzero multiple of 64. */ static PRISM_INLINE void -pm_state_stack_push(pm_state_stack_t *stack, bool value) { - *stack = (*stack << 1) | (value & 1); +pm_state_stack_push(pm_arena_t *arena, pm_state_stack_t *stack, bool value) { + if (PRISM_UNLIKELY(stack->depth && !(stack->depth & 63))) { + pm_state_stack_spill(arena, stack); + } + + stack->top = (stack->top << 1) | value; + stack->depth++; } /** - * Pops a value off the stack. + * Pops a value off the stack. When that empties the inline word, the newest + * spilled word moves back into it, so the inline word is empty only when the + * whole stack is. */ static PRISM_INLINE void pm_state_stack_pop(pm_state_stack_t *stack) { - *stack >>= 1; + /* A closing delimiter without a matching opening one is a syntax error that + * the parser reports and recovers from, but the lexer still pops for it, so + * popping an empty stack has to leave it empty. */ + if (PRISM_UNLIKELY(stack->depth == 0)) return; + + stack->top >>= 1; + stack->depth--; + + if (PRISM_UNLIKELY(stack->depth && !(stack->depth & 63))) { + stack->top = stack->spill[(stack->depth >> 6) - 1]; + } } /** - * Returns the value at the top of the stack. + * Returns the value at the top of the stack. An empty stack reads as false, + * because its inline word is 0. */ static PRISM_INLINE bool pm_state_stack_p(const pm_state_stack_t *stack) { - return *stack & 1; + return stack->top & 1; } +/** + * The accepts-block stack stores the negation of each value, so that an empty + * stack (which reads as false) means blocks are accepted. + */ static PRISM_INLINE void pm_accepts_block_stack_push(pm_parser_t *parser, bool value) { - // Use the negation of the value to prevent stack overflow. - pm_state_stack_push(&parser->accepts_block_stack, !value); + pm_state_stack_push(&parser->scratch_arena, &parser->accepts_block_stack, !value); } static PRISM_INLINE void @@ -7313,7 +7353,7 @@ pm_accepts_block_stack_p(pm_parser_t *parser) { static PRISM_INLINE void pm_do_loop_stack_push(pm_parser_t *parser, bool value) { - pm_state_stack_push(&parser->do_loop_stack, value); + pm_state_stack_push(&parser->scratch_arena, &parser->do_loop_stack, value); } static PRISM_INLINE void @@ -23167,8 +23207,8 @@ pm_parser_init(pm_arena_t *arena, pm_parser_t *parser, const uint8_t *source, si .enclosure_nesting = 0, .lambda_enclosure_nesting = -1, .brace_nesting = 0, - .do_loop_stack = 0, - .accepts_block_stack = 0, + .do_loop_stack = { 0 }, + .accepts_block_stack = { 0 }, .lex_modes = { .current = &parser->lex_modes.base, .base = { .mode = PM_LEX_DEFAULT }, diff --git a/test/prism/result/delimiter_nesting_test.rb b/test/prism/result/delimiter_nesting_test.rb new file mode 100644 index 0000000000..6dbfcaa6e1 --- /dev/null +++ b/test/prism/result/delimiter_nesting_test.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module Prism + # The lexer tracks whether a `do` closes a loop predicate and whether a `do` + # can open a block on stacks with one frame per open delimiter. These tests + # nest delimiters around and across the 64-frame words those stacks spill + # into, and check that the frame beneath the delimiters is still read once + # they close. + class DelimiterNestingTest < TestCase + DEPTHS = [1, 31, 32, 33, 63, 64, 65, 127, 128, 129, 200, 1000] + + def test_loop_predicate + DEPTHS.each do |depth| + source = "while #{"(" * depth}x#{")" * depth} do end" + assert_loop(WhileNode, source) + end + end + + def test_mixed_delimiters + DEPTHS.each do |depth| + opening = "([" * depth + closing = "])" * depth + source = "until #{opening}x#{closing} do end" + assert_loop(UntilNode, source) + end + end + + def test_nested_loop_predicate + DEPTHS.each do |depth| + inner = "while #{"(" * depth}y#{")" * depth} do end" + source = "while #{"(" * depth}#{inner}#{")" * depth} do end" + + node = assert_loop(WhileNode, source) + assert_kind_of WhileNode, node.predicate.breadth_first_search { |child| child.is_a?(WhileNode) } + end + end + + def test_block_inside_loop_predicate + DEPTHS.each do |depth| + source = "while #{"(" * depth}foo do end#{")" * depth} do end" + + node = assert_loop(WhileNode, source) + call = node.predicate.breadth_first_search { |child| child.is_a?(CallNode) } + assert_kind_of BlockNode, call.block + end + end + + # A closing delimiter without a matching opening one still pops both + # stacks, so these pop an empty stack, some of them more than 64 times in + # a row, and check that the source still parses with errors. + def test_unmatched_closers + sources = [")", "foo]", "}", "def foo; ); end", "while x) do end", ")" * 100, "]" * 100, "}" * 100] + + sources.each do |source| + refute_empty Prism.parse(source).errors, source + end + end + + private + + def assert_loop(type, source) + result = Prism.parse(source) + assert_empty result.errors, source + + node = result.value.statements.body.first + assert_kind_of type, node + node + end + end +end