|
| 1 | +#include "parts.h" |
| 2 | + |
| 3 | +#include "../../Parser/tokenizer/cursor.h" |
| 4 | + |
| 5 | +static int |
| 6 | +check(int condition, const char *message) |
| 7 | +{ |
| 8 | + if (condition) { |
| 9 | + return 0; |
| 10 | + } |
| 11 | + PyErr_SetString(PyExc_AssertionError, message); |
| 12 | + return -1; |
| 13 | +} |
| 14 | + |
| 15 | +static int |
| 16 | +check_system_error(int failed, const char *message) |
| 17 | +{ |
| 18 | + if (!failed || !PyErr_ExceptionMatches(PyExc_SystemError)) { |
| 19 | + PyErr_SetString(PyExc_AssertionError, message); |
| 20 | + return -1; |
| 21 | + } |
| 22 | + PyErr_Clear(); |
| 23 | + return 0; |
| 24 | +} |
| 25 | + |
| 26 | +static int |
| 27 | +same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right) |
| 28 | +{ |
| 29 | + return left->source == right->source && |
| 30 | + left->pos == right->pos && |
| 31 | + left->line_start == right->line_start && |
| 32 | + left->line_end == right->line_end && |
| 33 | + left->lineno == right->lineno; |
| 34 | +} |
| 35 | + |
| 36 | +static PyObject * |
| 37 | +test_tokenizer_source(PyObject *Py_UNUSED(module), |
| 38 | + PyObject *Py_UNUSED(args)) |
| 39 | +{ |
| 40 | + _PyTok_SourceText source; |
| 41 | + _PyTok_SourceInit(&source); |
| 42 | + |
| 43 | + _PyTok_Loc loc; |
| 44 | + _PyTok_Line line; |
| 45 | + if (check(_PyTok_SourceLocation( |
| 46 | + &source, 0, _PYTOK_AFFINITY_RIGHT, &loc) == 0, |
| 47 | + "cannot locate empty source") < 0 || |
| 48 | + check(loc.lineno == 1 && loc.byte_col == 0, |
| 49 | + "wrong empty source location") < 0 || |
| 50 | + check(_PyTok_SourceLine(&source, 1, &line) == 0, |
| 51 | + "cannot find empty source line") < 0 || |
| 52 | + check(line.start == 0 && line.end == 0, |
| 53 | + "wrong empty source line") < 0 || |
| 54 | + check_system_error( |
| 55 | + _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, |
| 56 | + "accepted empty source line") < 0 || |
| 57 | + check_system_error( |
| 58 | + _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0, |
| 59 | + "accepted multiple source lines") < 0 || |
| 60 | + check_system_error( |
| 61 | + _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0, |
| 62 | + "accepted missing implicit newline") < 0) { |
| 63 | + goto error; |
| 64 | + } |
| 65 | + |
| 66 | + if (check(_PyTok_SourceAppendLine(&source, "alpha\n", 6, 0) == 0, |
| 67 | + "wrong first source offset") < 0 || |
| 68 | + check(_PyTok_SourceAppendLine( |
| 69 | + &source, "\xce\xb2\n", 3, 1) == 6, |
| 70 | + "wrong second source offset") < 0 || |
| 71 | + check(_PyTok_SourceAppendLine( |
| 72 | + &source, "nul\0x\n", 6, 0) == 9, |
| 73 | + "wrong third source offset") < 0) { |
| 74 | + goto error; |
| 75 | + } |
| 76 | + |
| 77 | + int marker_line = 257; |
| 78 | + int final_line = 300; |
| 79 | + _PyTok_Off marker_start = -1; |
| 80 | + for (int lineno = 4; lineno <= final_line; lineno++) { |
| 81 | + const char *text = lineno == marker_line ? "marker\n" : "x\n"; |
| 82 | + Py_ssize_t len = (Py_ssize_t)strlen(text); |
| 83 | + _PyTok_Off start = _PyTok_SourceAppendLine( |
| 84 | + &source, text, len, lineno == final_line); |
| 85 | + if (start < 0) { |
| 86 | + goto error; |
| 87 | + } |
| 88 | + if (lineno == marker_line) { |
| 89 | + marker_start = start; |
| 90 | + } |
| 91 | + } |
| 92 | + |
| 93 | + if (check(source.nlines == final_line, "wrong source line count") < 0 || |
| 94 | + check(_PyTok_SourceLine(&source, marker_line, &line) == 0, |
| 95 | + "cannot find late source line") < 0 || |
| 96 | + check(line.start == marker_start && |
| 97 | + line.end == marker_start + 7, |
| 98 | + "wrong late source line") < 0 || |
| 99 | + check(!line.implicit_newline && !line.contains_nul, |
| 100 | + "wrong late source flags") < 0 || |
| 101 | + check(_PyTok_SourceLine(&source, 2, &line) == 0, |
| 102 | + "cannot find second source line") < 0 || |
| 103 | + check(line.start == 6 && line.end == 9 && |
| 104 | + line.implicit_newline && !line.contains_nul, |
| 105 | + "wrong second source line") < 0 || |
| 106 | + check(!_PyTok_SourceLineIsImplicit(&source, 1) && |
| 107 | + _PyTok_SourceLineIsImplicit(&source, 2), |
| 108 | + "wrong early implicit newline flags") < 0 || |
| 109 | + check(_PyTok_SourceLine(&source, 3, &line) == 0, |
| 110 | + "cannot find third source line") < 0 || |
| 111 | + check(line.contains_nul, "missing null byte flag") < 0 || |
| 112 | + check(_PyTok_SourceLine(&source, final_line, &line) == 0, |
| 113 | + "cannot find final source line") < 0 || |
| 114 | + check(line.implicit_newline && |
| 115 | + _PyTok_SourceLineIsImplicit(&source, final_line), |
| 116 | + "missing late implicit newline flag") < 0) { |
| 117 | + goto error; |
| 118 | + } |
| 119 | + |
| 120 | + Py_ssize_t view_len; |
| 121 | + const char *view = _PyTok_SourceSpanView( |
| 122 | + &source, _PyTok_SpanFromBounds(6, 8), &view_len); |
| 123 | + if (check(view != NULL && view_len == 2 && |
| 124 | + memcmp(view, "\xce\xb2", 2) == 0, |
| 125 | + "wrong source span view") < 0 || |
| 126 | + check(_PyTok_SourceLocation( |
| 127 | + &source, marker_start, |
| 128 | + _PYTOK_AFFINITY_LEFT, &loc) == 0, |
| 129 | + "cannot locate left line boundary") < 0 || |
| 130 | + check(loc.lineno == marker_line - 1 && loc.byte_col == 2, |
| 131 | + "wrong left boundary location") < 0 || |
| 132 | + check(_PyTok_SourceLocation( |
| 133 | + &source, marker_start, |
| 134 | + _PYTOK_AFFINITY_RIGHT, &loc) == 0, |
| 135 | + "cannot locate right line boundary") < 0 || |
| 136 | + check(loc.lineno == marker_line && loc.byte_col == 0, |
| 137 | + "wrong right boundary location") < 0 || |
| 138 | + check(_PyTok_SourceLocation( |
| 139 | + &source, marker_start + 1, |
| 140 | + _PYTOK_AFFINITY_RIGHT, &loc) == 0, |
| 141 | + "cannot locate late source byte") < 0 || |
| 142 | + check(loc.lineno == marker_line && loc.byte_col == 1, |
| 143 | + "wrong late source location") < 0) { |
| 144 | + goto error; |
| 145 | + } |
| 146 | + |
| 147 | + if (check(_PyTok_SourceLocation( |
| 148 | + &source, source.len, _PYTOK_AFFINITY_LEFT, &loc) == 0, |
| 149 | + "cannot locate left EOF") < 0 || |
| 150 | + check(loc.lineno == final_line && loc.byte_col == 2, |
| 151 | + "wrong left EOF location") < 0 || |
| 152 | + check(_PyTok_SourceLocation( |
| 153 | + &source, source.len, |
| 154 | + _PYTOK_AFFINITY_RIGHT, &loc) == 0, |
| 155 | + "cannot locate right EOF") < 0 || |
| 156 | + check(loc.lineno == final_line + 1 && loc.byte_col == 0, |
| 157 | + "wrong right EOF location") < 0 || |
| 158 | + check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0, |
| 159 | + "cannot find virtual EOF line") < 0 || |
| 160 | + check(line.start == source.len && line.end == source.len, |
| 161 | + "wrong virtual EOF line") < 0 || |
| 162 | + check(!_PyTok_SourceLineIsImplicit(&source, 0) && |
| 163 | + !_PyTok_SourceLineIsImplicit( |
| 164 | + &source, final_line + 1), |
| 165 | + "virtual or invalid line is implicit") < 0) { |
| 166 | + goto error; |
| 167 | + } |
| 168 | + |
| 169 | + view = _PyTok_SourceSpanView( |
| 170 | + &source, _PyTok_SpanFromBounds(0, source.len + 1), &view_len); |
| 171 | + if (check_system_error(view == NULL, "accepted invalid source span") < 0 || |
| 172 | + check_system_error( |
| 173 | + _PyTok_SourceLocation( |
| 174 | + &source, source.len + 1, |
| 175 | + _PYTOK_AFFINITY_RIGHT, &loc) < 0, |
| 176 | + "accepted invalid source offset") < 0 || |
| 177 | + check_system_error( |
| 178 | + _PyTok_SourceLine(&source, final_line + 2, &line) < 0, |
| 179 | + "accepted invalid source line") < 0) { |
| 180 | + goto error; |
| 181 | + } |
| 182 | + |
| 183 | + _PyTok_SourceClear(&source); |
| 184 | + _PyTok_SourceInit(&source); |
| 185 | + if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 || |
| 186 | + check_system_error( |
| 187 | + _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0, |
| 188 | + "appended after unterminated source line") < 0 || |
| 189 | + check(_PyTok_SourceLocation( |
| 190 | + &source, source.len, |
| 191 | + _PYTOK_AFFINITY_RIGHT, &loc) == 0, |
| 192 | + "cannot locate unterminated EOF") < 0 || |
| 193 | + check(loc.lineno == 1 && loc.byte_col == 4, |
| 194 | + "wrong unterminated EOF location") < 0) { |
| 195 | + goto error; |
| 196 | + } |
| 197 | + |
| 198 | + _PyTok_SourceClear(&source); |
| 199 | + Py_RETURN_NONE; |
| 200 | + |
| 201 | +error: |
| 202 | + _PyTok_SourceClear(&source); |
| 203 | + return NULL; |
| 204 | +} |
| 205 | + |
| 206 | +static PyObject * |
| 207 | +test_tokenizer_cursor(PyObject *Py_UNUSED(module), |
| 208 | + PyObject *Py_UNUSED(args)) |
| 209 | +{ |
| 210 | + _PyTok_SourceText source; |
| 211 | + _PyTok_SourceInit(&source); |
| 212 | + if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 || |
| 213 | + _PyTok_SourceAppendLine(&source, "cd\n", 3, 0) < 0) { |
| 214 | + goto error; |
| 215 | + } |
| 216 | + |
| 217 | + _PyTok_Cursor cursor; |
| 218 | + _PyTok_CursorInit(&cursor, &source); |
| 219 | + if (_PyTok_CursorSetOffset(&cursor, source.len) < 0 || |
| 220 | + check(cursor.lineno == 3 && cursor.pos == source.len, |
| 221 | + "wrong cursor at virtual EOF") < 0 || |
| 222 | + _PyTok_CursorSetLine(&cursor, 1) < 0) { |
| 223 | + goto error; |
| 224 | + } |
| 225 | + |
| 226 | + char large[BUFSIZ + 1]; |
| 227 | + memset(large, 'z', sizeof(large)); |
| 228 | + large[sizeof(large) - 1] = '\n'; |
| 229 | + if (_PyTok_SourceAppendLine(&source, large, sizeof(large), 0) < 0) { |
| 230 | + goto error; |
| 231 | + } |
| 232 | + |
| 233 | + if (check(_PyTok_CursorPeek(&cursor, 0) == 'a', |
| 234 | + "wrong cursor peek after relocation") < 0 || |
| 235 | + check(_PyTok_CursorPeek(&cursor, 1) == 'b', |
| 236 | + "wrong distant cursor peek") < 0 || |
| 237 | + check(_PyTok_CursorAdvance(&cursor) == 'a', |
| 238 | + "wrong first cursor byte") < 0 || |
| 239 | + check(_PyTok_CursorAdvance(&cursor) == 'b', |
| 240 | + "wrong second cursor byte") < 0 || |
| 241 | + check(_PyTok_CursorAdvance(&cursor) == '\n', |
| 242 | + "wrong final cursor byte") < 0 || |
| 243 | + check(_PyTok_CursorAdvance(&cursor) == EOF, |
| 244 | + "cursor advanced past line") < 0 || |
| 245 | + check(_PyTok_CursorSetOffset(&cursor, 2) == 0, |
| 246 | + "cannot seek cursor offset") < 0 || |
| 247 | + check(_PyTok_CursorAdvance(&cursor) == '\n', |
| 248 | + "wrong cursor byte after seek") < 0 || |
| 249 | + check(_PyTok_CursorSetOffset(&cursor, 3) == 0, |
| 250 | + "cannot seek line boundary") < 0 || |
| 251 | + check(cursor.lineno == 2 && cursor.line_start == 3 && |
| 252 | + _PyTok_CursorAdvance(&cursor) == 'c', |
| 253 | + "wrong cursor at line boundary") < 0 || |
| 254 | + check(_PyTok_CursorSetLine(&cursor, 3) == 0, |
| 255 | + "cannot advance cursor to final line") < 0 || |
| 256 | + check(cursor.line_start == 6 && |
| 257 | + _PyTok_CursorAdvance(&cursor) == 'z', |
| 258 | + "wrong cursor byte on final line") < 0) { |
| 259 | + goto error; |
| 260 | + } |
| 261 | + |
| 262 | + _PyTok_Cursor saved = cursor; |
| 263 | + if (check_system_error( |
| 264 | + _PyTok_CursorSetOffset(&cursor, source.len + 1) < 0, |
| 265 | + "accepted invalid cursor offset") < 0 || |
| 266 | + check(same_cursor(&cursor, &saved), |
| 267 | + "invalid offset changed cursor") < 0 || |
| 268 | + check_system_error( |
| 269 | + _PyTok_CursorSetLine(&cursor, source.nlines + 2) < 0, |
| 270 | + "accepted invalid cursor line") < 0 || |
| 271 | + check(same_cursor(&cursor, &saved), |
| 272 | + "invalid line changed cursor") < 0 || |
| 273 | + check(_PyTok_CursorSetOffset(&cursor, source.len) == 0, |
| 274 | + "cannot set cursor to EOF") < 0 || |
| 275 | + check(cursor.lineno == 4 && cursor.pos == source.len, |
| 276 | + "wrong cursor at EOF") < 0) { |
| 277 | + goto error; |
| 278 | + } |
| 279 | + |
| 280 | +#if SIZEOF_VOID_P > 4 |
| 281 | + char byte = 0; |
| 282 | + _PyTok_SourceText huge_source = { |
| 283 | + .bytes = &byte, |
| 284 | + .len = (_PyTok_Off)INT_MAX + 1, |
| 285 | + }; |
| 286 | + _PyTok_Cursor huge_cursor = { |
| 287 | + .source = &huge_source, |
| 288 | + .pos = INT_MAX, |
| 289 | + .line_end = (_PyTok_Off)INT_MAX + 1, |
| 290 | + .lineno = 1, |
| 291 | + }; |
| 292 | + if (check(_PyTok_CursorAdvance(&huge_cursor) == EOF && |
| 293 | + huge_cursor.pos == INT_MAX, |
| 294 | + "cursor advanced past maximum column") < 0) { |
| 295 | + goto error; |
| 296 | + } |
| 297 | +#endif |
| 298 | + |
| 299 | + _PyTok_SourceClear(&source); |
| 300 | + Py_RETURN_NONE; |
| 301 | + |
| 302 | +error: |
| 303 | + _PyTok_SourceClear(&source); |
| 304 | + return NULL; |
| 305 | +} |
| 306 | + |
| 307 | +static PyMethodDef test_methods[] = { |
| 308 | + {"test_tokenizer_source", test_tokenizer_source, METH_NOARGS}, |
| 309 | + {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS}, |
| 310 | + {NULL}, |
| 311 | +}; |
| 312 | + |
| 313 | +int |
| 314 | +_PyTestInternalCapi_Init_Tokenizer(PyObject *module) |
| 315 | +{ |
| 316 | + return PyModule_AddFunctions(module, test_methods); |
| 317 | +} |
0 commit comments