Skip to content

Commit 072e00d

Browse files
tausbntausbn
authored
Merge pull request #22728 from github/tausbn/python-extractor-support-python-3.15
Python: Add extractor support for Python 3.15
2 parents 4f40d24 + fa4ce46 commit 072e00d

18 files changed

Lines changed: 14403 additions & 13889 deletions

File tree

‎python/extractor/extending-the-parser.md‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,21 +43,28 @@ This grammar can be found in the `grammar.js` file in the `tsg-python/tsp` subdi
4343
Note that whenever changes are made to `grammar.js`, you must regenerate the parser files by running
4444

4545
```sh
46-
tree-sitter generate
46+
tree-sitter generate --abi 14
4747
```
4848

4949
inside the `tsp` directory.
5050
You'll need to install the `tree-sitter` CLI in order to run this command.
5151
One way to install it is to use `cargo`:
5252

5353
```sh
54-
cargo install tree-sitter-cli
54+
cargo install tree-sitter-cli --version 0.25.10 --locked
5555
```
5656

5757
(This presupposes you have `cargo` available, but you'll need this anyway when compiling `tsg-python`.)
5858

5959
Once the parser files have been regenerated, they'll get picked up automatically when `tsg-python` is rebuilt.
6060

61+
The identifier rule supplements the generator's Unicode 16 properties with the
62+
Unicode 17 `XID_Start` and `XID_Continue` additions from the Unicode Character
63+
Database. Keep both sets of additions in sync when updating Unicode support.
64+
Once the generator includes Unicode 17, these additions can be removed.
65+
The `unicode_identifiers_new.py` parser fixture and `unicode17-identifiers` QL test
66+
cover extraction independently of the host Python interpreter's Unicode version.
67+
6168
> Pro-tip: When you're done with your parser changes, and go to commit these to a branch, put the autogenerated files in their own commit.
6269
> This makes it easier to review the changes, and if you need to go back and regenerate the files again, it's easy to modify just that commit.
6370

‎python/extractor/semmle/util.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
#Semantic version of extractor.
1212
#Update this if any changes are made
13-
VERSION = "7.1.11"
13+
VERSION = "7.2.0"
1414

1515
PY_EXTENSIONS = ".py", ".pyw"
1616

‎python/extractor/tests/parser/match_new.expected‎

Lines changed: 210 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Module: [1, 0] - [43, 0]
1+
Module: [1, 0] - [61, 0]
22
body: [
33
Match: [1, 0] - [3, 19]
44
subject:
@@ -85,7 +85,7 @@ Module: [1, 0] - [43, 0]
8585
literal:
8686
BinOp: [8, 9] - [8, 18]
8787
left:
88-
UnaryOp: [8, 10] - [8, 11]
88+
UnaryOp: [8, 9] - [8, 11]
8989
op: USub
9090
operand:
9191
Num: [8, 10] - [8, 11]
@@ -104,7 +104,7 @@ Module: [1, 0] - [43, 0]
104104
pattern:
105105
MatchLiteralPattern: [10, 9] - [10, 11]
106106
literal:
107-
UnaryOp: [10, 10] - [10, 11]
107+
UnaryOp: [10, 9] - [10, 11]
108108
op: USub
109109
operand:
110110
Num: [10, 10] - [10, 11]
@@ -131,7 +131,7 @@ Module: [1, 0] - [43, 0]
131131
literal:
132132
BinOp: [14, 9] - [14, 16]
133133
left:
134-
UnaryOp: [14, 10] - [14, 13]
134+
UnaryOp: [14, 9] - [14, 13]
135135
op: USub
136136
operand:
137137
Num: [14, 10] - [14, 13]
@@ -379,4 +379,210 @@ Module: [1, 0] - [43, 0]
379379
Pass: [42, 8] - [42, 12]
380380
]
381381
]
382+
Match: [44, 0] - [60, 12]
383+
subject:
384+
Name: [44, 6] - [44, 11]
385+
variable: Variable('value', None)
386+
ctx: Load
387+
cases: [
388+
Case: [45, 4] - [46, 12]
389+
pattern:
390+
MatchLiteralPattern: [45, 9] - [45, 11]
391+
literal:
392+
UnaryOp: [45, 9] - [45, 11]
393+
op: UAdd
394+
operand:
395+
Num: [45, 10] - [45, 11]
396+
n: 1
397+
text: '1'
398+
guard: None
399+
body: [
400+
Pass: [46, 8] - [46, 12]
401+
]
402+
Case: [47, 4] - [48, 12]
403+
pattern:
404+
MatchLiteralPattern: [47, 9] - [47, 13]
405+
literal:
406+
UnaryOp: [47, 9] - [47, 13]
407+
op: UAdd
408+
operand:
409+
Num: [47, 10] - [47, 13]
410+
n: 1.5
411+
text: '1.5'
412+
guard: None
413+
body: [
414+
Pass: [48, 8] - [48, 12]
415+
]
416+
Case: [49, 4] - [50, 12]
417+
pattern:
418+
MatchLiteralPattern: [49, 9] - [49, 12]
419+
literal:
420+
UnaryOp: [49, 9] - [49, 12]
421+
op: UAdd
422+
operand:
423+
Num: [49, 10] - [49, 12]
424+
n: 2j
425+
text: '2j'
426+
guard: None
427+
body: [
428+
Pass: [50, 8] - [50, 12]
429+
]
430+
Case: [51, 4] - [52, 12]
431+
pattern:
432+
MatchLiteralPattern: [51, 9] - [51, 14]
433+
literal:
434+
BinOp: [51, 9] - [51, 14]
435+
left:
436+
UnaryOp: [51, 9] - [51, 11]
437+
op: UAdd
438+
operand:
439+
Num: [51, 10] - [51, 11]
440+
n: 1
441+
text: '1'
442+
op: Add
443+
right:
444+
Num: [51, 12] - [51, 14]
445+
n: 2j
446+
text: '2j'
447+
guard: None
448+
body: [
449+
Pass: [52, 8] - [52, 12]
450+
]
451+
Case: [53, 4] - [54, 12]
452+
pattern:
453+
MatchLiteralPattern: [53, 9] - [53, 14]
454+
literal:
455+
BinOp: [53, 9] - [53, 14]
456+
left:
457+
UnaryOp: [53, 9] - [53, 11]
458+
op: UAdd
459+
operand:
460+
Num: [53, 10] - [53, 11]
461+
n: 1
462+
text: '1'
463+
op: Sub
464+
right:
465+
Num: [53, 12] - [53, 14]
466+
n: 2j
467+
text: '2j'
468+
guard: None
469+
body: [
470+
Pass: [54, 8] - [54, 12]
471+
]
472+
Case: [55, 4] - [56, 12]
473+
pattern:
474+
MatchOrPattern: [55, 9] - [55, 30]
475+
patterns: [
476+
MatchLiteralPattern: [55, 9] - [55, 14]
477+
literal:
478+
UnaryOp: [55, 9] - [55, 14]
479+
op: UAdd
480+
operand:
481+
Num: [55, 10] - [55, 14]
482+
n: 16
483+
text: '0x10'
484+
MatchLiteralPattern: [55, 17] - [55, 22]
485+
literal:
486+
UnaryOp: [55, 17] - [55, 22]
487+
op: UAdd
488+
operand:
489+
Num: [55, 18] - [55, 22]
490+
n: 8
491+
text: '0o10'
492+
MatchLiteralPattern: [55, 25] - [55, 30]
493+
literal:
494+
UnaryOp: [55, 25] - [55, 30]
495+
op: UAdd
496+
operand:
497+
Num: [55, 26] - [55, 30]
498+
n: 2
499+
text: '0b10'
500+
]
501+
guard: None
502+
body: [
503+
Pass: [56, 8] - [56, 12]
504+
]
505+
Case: [57, 4] - [58, 12]
506+
pattern:
507+
MatchSequencePattern: [57, 9] - [57, 19]
508+
patterns: [
509+
MatchLiteralPattern: [57, 10] - [57, 12]
510+
literal:
511+
UnaryOp: [57, 10] - [57, 12]
512+
op: UAdd
513+
operand:
514+
Num: [57, 11] - [57, 12]
515+
n: 0
516+
text: '0'
517+
MatchLiteralPattern: [57, 14] - [57, 18]
518+
literal:
519+
UnaryOp: [57, 14] - [57, 18]
520+
op: USub
521+
operand:
522+
Num: [57, 15] - [57, 18]
523+
n: 0.0
524+
text: '0.0'
525+
]
526+
guard: None
527+
body: [
528+
Pass: [58, 8] - [58, 12]
529+
]
530+
Case: [59, 4] - [60, 12]
531+
pattern:
532+
MatchMappingPattern: [59, 9] - [59, 33]
533+
mappings: [
534+
MatchKeyValuePattern: [59, 10] - [59, 15]
535+
key:
536+
MatchLiteralPattern: [59, 10] - [59, 12]
537+
literal:
538+
UnaryOp: [59, 10] - [59, 12]
539+
op: UAdd
540+
operand:
541+
Num: [59, 11] - [59, 12]
542+
n: 1
543+
text: '1'
544+
value:
545+
MatchCapturePattern: [59, 14] - [59, 15]
546+
variable:
547+
Name: [59, 14] - [59, 15]
548+
variable: Variable('x', None)
549+
ctx: Store
550+
MatchKeyValuePattern: [59, 17] - [59, 24]
551+
key:
552+
MatchLiteralPattern: [59, 17] - [59, 21]
553+
literal:
554+
UnaryOp: [59, 17] - [59, 21]
555+
op: UAdd
556+
operand:
557+
Num: [59, 18] - [59, 21]
558+
n: 1.5
559+
text: '1.5'
560+
value:
561+
MatchCapturePattern: [59, 23] - [59, 24]
562+
variable:
563+
Name: [59, 23] - [59, 24]
564+
variable: Variable('y', None)
565+
ctx: Store
566+
MatchKeyValuePattern: [59, 26] - [59, 32]
567+
key:
568+
MatchLiteralPattern: [59, 26] - [59, 29]
569+
literal:
570+
UnaryOp: [59, 26] - [59, 29]
571+
op: UAdd
572+
operand:
573+
Num: [59, 27] - [59, 29]
574+
n: 2j
575+
text: '2j'
576+
value:
577+
MatchCapturePattern: [59, 31] - [59, 32]
578+
variable:
579+
Name: [59, 31] - [59, 32]
580+
variable: Variable('z', None)
581+
ctx: Store
582+
]
583+
guard: None
584+
body: [
585+
Pass: [60, 8] - [60, 12]
586+
]
587+
]
382588
]

‎python/extractor/tests/parser/match_new.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,3 +40,21 @@ def soft_keywords():
4040
match z:
4141
case w:
4242
pass
43+
44+
match value:
45+
case +1:
46+
pass
47+
case +1.5:
48+
pass
49+
case +2j:
50+
pass
51+
case +1+2j:
52+
pass
53+
case +1-2j:
54+
pass
55+
case +0x10 | +0o10 | +0b10:
56+
pass
57+
case [+0, -0.0]:
58+
pass
59+
case {+1: x, +1.5: y, +2j: z}:
60+
pass

0 commit comments

Comments
 (0)