Skip to content

Commit 8a96ebd

Browse files
pablogsalpablogsal
committed
gh-153569: retain decoded reader input in source text
1 parent cc82cac commit 8a96ebd

6 files changed

Lines changed: 100 additions & 91 deletions

File tree

‎Parser/lexer/buffer.c‎

Lines changed: 32 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,45 @@
11
#include "Python.h"
2-
#include "errcode.h"
3-
2+
#include "buffer.h"
43
#include "state.h"
54

6-
/* Traverse and remember all f-string buffers, in order to be able to restore
7-
them after reallocating tok->buf */
85
void
9-
_PyLexer_remember_fstring_buffers(struct tok_state *tok)
6+
_PyLexer_SnapshotBuffer(struct tok_state *tok, const char *base,
7+
_PyLexer_BufferSnapshot *snapshot)
108
{
11-
int index;
12-
tokenizer_mode *mode;
13-
14-
for (index = tok->tok_mode_stack_index; index >= 0; --index) {
15-
mode = &(tok->tok_mode_stack[index]);
9+
snapshot->buf = tok->buf - base;
10+
snapshot->cur = tok->cur - tok->buf;
11+
snapshot->inp = tok->inp - tok->buf;
12+
snapshot->start = tok->start == NULL ? -1 : tok->start - tok->buf;
13+
snapshot->line_start = tok->line_start == NULL
14+
? -1 : tok->line_start - tok->buf;
15+
snapshot->multi_line_start = tok->multi_line_start == NULL
16+
? -1 : tok->multi_line_start - tok->buf;
17+
for (int index = tok->tok_mode_stack_index; index >= 0; --index) {
18+
tokenizer_mode *mode = &tok->tok_mode_stack[index];
1619
mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
17-
mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf;
20+
mode->multi_line_start_offset = mode->multi_line_start == NULL
21+
? -1 : mode->multi_line_start - tok->buf;
1822
}
1923
}
2024

21-
/* Traverse and restore all f-string buffers after reallocating tok->buf */
2225
void
23-
_PyLexer_restore_fstring_buffers(struct tok_state *tok)
24-
{
25-
int index;
26-
tokenizer_mode *mode;
27-
28-
for (index = tok->tok_mode_stack_index; index >= 0; --index) {
29-
mode = &(tok->tok_mode_stack[index]);
30-
mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset;
31-
mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset;
32-
}
33-
}
34-
35-
int
36-
_PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size)
26+
_PyLexer_RestoreBuffer(struct tok_state *tok, char *base,
27+
const _PyLexer_BufferSnapshot *snapshot)
3728
{
38-
Py_ssize_t cur = tok->cur - tok->buf;
39-
Py_ssize_t oldsize = tok->inp - tok->buf;
40-
Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1);
41-
if (newsize > tok->end - tok->buf) {
42-
char *newbuf = tok->buf;
43-
Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf;
44-
Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf;
45-
Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf;
46-
_PyLexer_remember_fstring_buffers(tok);
47-
newbuf = (char *)PyMem_Realloc(newbuf, newsize);
48-
if (newbuf == NULL) {
49-
tok->done = E_NOMEM;
50-
return 0;
51-
}
52-
tok->buf = newbuf;
53-
tok->cur = tok->buf + cur;
54-
tok->inp = tok->buf + oldsize;
55-
tok->end = tok->buf + newsize;
56-
tok->start = start < 0 ? NULL : tok->buf + start;
57-
tok->line_start = line_start < 0 ? NULL : tok->buf + line_start;
58-
tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start;
59-
_PyLexer_restore_fstring_buffers(tok);
29+
tok->buf = base + snapshot->buf;
30+
tok->cur = tok->buf + snapshot->cur;
31+
tok->inp = tok->buf + snapshot->inp;
32+
tok->start = snapshot->start < 0
33+
? NULL : tok->buf + snapshot->start;
34+
tok->line_start = snapshot->line_start < 0
35+
? NULL : tok->buf + snapshot->line_start;
36+
tok->multi_line_start = snapshot->multi_line_start < 0
37+
? NULL : tok->buf + snapshot->multi_line_start;
38+
for (int index = tok->tok_mode_stack_index; index >= 0; --index) {
39+
tokenizer_mode *mode = &tok->tok_mode_stack[index];
40+
mode->start = mode->start_offset < 0
41+
? NULL : tok->buf + mode->start_offset;
42+
mode->multi_line_start = mode->multi_line_start_offset < 0
43+
? NULL : tok->buf + mode->multi_line_start_offset;
6044
}
61-
return 1;
6245
}

‎Parser/lexer/buffer.h‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,20 @@
33

44
#include "pyport.h"
55

6-
void _PyLexer_remember_fstring_buffers(struct tok_state *tok);
7-
void _PyLexer_restore_fstring_buffers(struct tok_state *tok);
8-
int _PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size);
6+
struct tok_state;
7+
8+
typedef struct {
9+
Py_ssize_t buf;
10+
Py_ssize_t cur;
11+
Py_ssize_t inp;
12+
Py_ssize_t start;
13+
Py_ssize_t line_start;
14+
Py_ssize_t multi_line_start;
15+
} _PyLexer_BufferSnapshot;
16+
17+
void _PyLexer_SnapshotBuffer(
18+
struct tok_state *, const char *, _PyLexer_BufferSnapshot *);
19+
void _PyLexer_RestoreBuffer(
20+
struct tok_state *, char *, const _PyLexer_BufferSnapshot *);
921

1022
#endif

‎Parser/lexer/state.c‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ _PyTokenizer_tok_new(void)
2626
tok->interactive_src_start = NULL;
2727
tok->interactive_src_end = NULL;
2828
tok->start = NULL;
29-
tok->end = NULL;
3029
tok->done = E_OK;
3130
tok->fp = NULL;
3231
tok->tabsize = TABSIZE;

‎Parser/lexer/state.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ struct tok_state {
7575
int fp_interactive; /* If the file descriptor is interactive */
7676
char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
7777
char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
78-
const char *end; /* End of input buffer if buf != NULL */
7978
const char *start; /* Start of current token if not NULL */
8079
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
8180
/* NB If done != E_OK, cur must be == inp!!! */

‎Parser/lexer/string.c‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
125125
{
126126
assert(tok->cur != NULL);
127127

128-
Py_ssize_t size = strlen(tok->cur);
128+
Py_ssize_t size = cur == 0
129+
? tok->inp - tok->cur : (Py_ssize_t)strlen(tok->cur);
129130
tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
130131

131132
switch (cur) {
@@ -142,7 +143,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
142143
goto error;
143144
}
144145
tok_mode->last_expr_buffer = new_buffer;
145-
strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size);
146+
memcpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size,
147+
tok->cur, size);
146148
tok_mode->last_expr_size += size;
147149
break;
148150
case '{':
@@ -155,7 +157,7 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
155157
}
156158
tok_mode->last_expr_size = size;
157159
tok_mode->last_expr_end = -1;
158-
strncpy(tok_mode->last_expr_buffer, tok->cur, size);
160+
memcpy(tok_mode->last_expr_buffer, tok->cur, size);
159161
break;
160162
case '}':
161163
case '!':

‎Parser/tokenizer/reader.c‎

Lines changed: 48 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,7 @@ _PyTok_ReaderFree(struct tok_state *tok)
2828
}
2929
PyMem_Free(reader->file_buffer);
3030
PyMem_Free(reader->decoded);
31-
if (reader->kind != _PYTOK_READER_PREPARED) {
32-
PyMem_Free(tok->buf);
33-
tok->buf = NULL;
34-
}
31+
tok->buf = NULL;
3532
PyMem_Free(reader);
3633
tok->reader = NULL;
3734
}
@@ -534,13 +531,13 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
534531
int prepared = tok->reader->kind == _PYTOK_READER_PREPARED;
535532
int reset_buffer = !prepared && tok->start == NULL && !INSIDE_FSTRING(tok);
536533

537-
if (reset_buffer && tok->reader->kind != _PYTOK_READER_INTERACTIVE) {
538-
tok->cur = tok->inp = tok->buf;
539-
}
540-
541534
_PyTok_Chunk chunk;
542535
_PyTok_ReadResult result = reader_next(tok, &chunk);
543536
if (result != _PYTOK_READ_LINE) {
537+
if (reset_buffer &&
538+
tok->reader->kind != _PYTOK_READER_INTERACTIVE) {
539+
tok->cur = tok->inp = tok->buf;
540+
}
544541
if (result == _PYTOK_READ_EOF) {
545542
tok->done = E_EOF;
546543
}
@@ -564,27 +561,48 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
564561
return 0;
565562
}
566563

567-
Py_ssize_t copy_len = chunk.len;
564+
Py_ssize_t scan_len = chunk.len;
568565
if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
569566
chunk.implicit_newline) {
570-
copy_len--;
571-
}
572-
if (reset_buffer && tok->reader->kind == _PYTOK_READER_INTERACTIVE) {
573-
tok->cur = tok->inp = tok->buf;
574-
}
575-
if (!prepared && !_PyLexer_tok_reserve_buf(tok, copy_len + 1)) {
576-
_PyTok_ChunkClear(&chunk);
577-
tok->input_error = 1;
578-
return 0;
579-
}
580-
if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
581-
_PyTok_SourceAppendLine(&tok->source, chunk.data, chunk.len,
582-
chunk.implicit_newline) < 0) {
583-
_PyTok_ChunkClear(&chunk);
584-
tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
585-
? E_NOMEM : E_ERROR;
586-
tok->input_error = 1;
587-
return 0;
567+
scan_len--;
568+
}
569+
if (!prepared) {
570+
int source_will_grow =
571+
chunk.len > tok->source.cap - tok->source.len - 1;
572+
_PyLexer_BufferSnapshot snapshot;
573+
if (!reset_buffer && source_will_grow) {
574+
_PyLexer_SnapshotBuffer(
575+
tok, tok->source.bytes, &snapshot);
576+
}
577+
_PyTok_Off source_start = _PyTok_SourceAppendLine(
578+
&tok->source, chunk.data, chunk.len,
579+
chunk.implicit_newline);
580+
if (source_start < 0) {
581+
_PyTok_ChunkClear(&chunk);
582+
tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
583+
? E_NOMEM : E_ERROR;
584+
tok->input_error = 1;
585+
return 0;
586+
}
587+
if (reset_buffer) {
588+
tok->buf = tok->cur = tok->source.bytes + source_start;
589+
tok->line_start = tok->buf;
590+
tok->start = NULL;
591+
tok->multi_line_start = NULL;
592+
}
593+
else if (source_will_grow) {
594+
_PyLexer_RestoreBuffer(
595+
tok, tok->source.bytes, &snapshot);
596+
}
597+
tok->inp = tok->source.bytes + source_start + scan_len;
598+
if (_PyTok_CursorSetLine(
599+
&tok->reader_cursor, tok->source.nlines) < 0) {
600+
_PyTok_ChunkClear(&chunk);
601+
tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
602+
? E_NOMEM : E_ERROR;
603+
tok->input_error = 1;
604+
return 0;
605+
}
588606
}
589607
if (tok->fp_interactive) {
590608
tok->interactive_src_start = tok->source.bytes;
@@ -596,11 +614,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
596614
}
597615
tok->inp = chunk.data + chunk.len;
598616
}
599-
else {
600-
memcpy(tok->inp, chunk.data, (size_t)copy_len);
601-
tok->inp += copy_len;
602-
*tok->inp = '\0';
603-
}
604617
tok->implicit_newline = chunk.implicit_newline;
605618

606619
if (!prepared && tok->tok_mode_stack_index &&
@@ -645,7 +658,9 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind)
645658
return NULL;
646659
}
647660
tok->cur = tok->inp = tok->buf;
648-
tok->end = tok->buf + BUFSIZ;
661+
tok->source.bytes = tok->buf;
662+
tok->source.cap = BUFSIZ;
663+
tok->source.bytes[0] = '\0';
649664
return tok;
650665
}
651666

@@ -663,7 +678,6 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input,
663678
return NULL;
664679
}
665680
tok->buf = tok->cur = tok->inp = tok->str;
666-
tok->end = tok->buf;
667681
return tok;
668682
}
669683

0 commit comments

Comments
 (0)