Skip to content

Commit c87180b

Browse files
tausbnCopilot
tausbn
and
Copilot
committed
Python: Support Unicode 17 identifiers
Supplement the generator's Unicode 16 identifier properties with the exact Unicode 17 additions, independently of the extraction host's Python version. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent fc13096 commit c87180b

8 files changed

Lines changed: 205 additions & 3 deletions

File tree

‎python/extractor/extending-the-parser.md‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,21 +43,28 @@ This grammar can be found in the `grammar.js` file in the `tsg-python/tsp` subdi
4343
Note that whenever changes are made to `grammar.js`, you must regenerate the parser files by running
4444

4545
```sh
46-
tree-sitter generate
46+
tree-sitter generate --abi 14
4747
```
4848

4949
inside the `tsp` directory.
5050
You'll need to install the `tree-sitter` CLI in order to run this command.
5151
One way to install it is to use `cargo`:
5252

5353
```sh
54-
cargo install tree-sitter-cli
54+
cargo install tree-sitter-cli --version 0.25.10 --locked
5555
```
5656

5757
(This presupposes you have `cargo` available, but you'll need this anyway when compiling `tsg-python`.)
5858

5959
Once the parser files have been regenerated, they'll get picked up automatically when `tsg-python` is rebuilt.
6060

61+
The identifier rule supplements the generator's Unicode 16 properties with the
62+
Unicode 17 `XID_Start` and `XID_Continue` additions from the Unicode Character
63+
Database. Keep both sets of additions in sync when updating Unicode support.
64+
Once the generator includes Unicode 17, these additions can be removed.
65+
The `unicode_identifiers_new.py` parser fixture and `unicode17-identifiers` QL test
66+
cover extraction independently of the host Python interpreter's Unicode version.
67+
6168
> Pro-tip: When you're done with your parser changes, and go to commit these to a branch, put the autogenerated files in their own commit.
6269
> This makes it easier to review the changes, and if you need to go back and regenerate the files again, it's easy to modify just that commit.
6370

‎python/extractor/tests/parser/unicode_identifiers_new.expected‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
Module: [2, 0] - [12, 0]
2+
body: [
3+
Assign: [2, 0] - [2, 7]
4+
targets: [
5+
Name: [2, 0] - [2, 3]
6+
variable: Variable('\u088f', None)
7+
ctx: Store
8+
]
9+
value:
10+
Num: [2, 6] - [2, 7]
11+
n: 1
12+
text: '1'
13+
Assign: [3, 0] - [3, 8]
14+
targets: [
15+
Name: [3, 0] - [3, 4]
16+
variable: Variable('\U0001e6c0', None)
17+
ctx: Store
18+
]
19+
value:
20+
Num: [3, 7] - [3, 8]
21+
n: 2
22+
text: '2'
23+
Assign: [4, 0] - [4, 8]
24+
targets: [
25+
Name: [4, 0] - [4, 4]
26+
variable: Variable('\U000323b0', None)
27+
ctx: Store
28+
]
29+
value:
30+
Num: [4, 7] - [4, 8]
31+
n: 3
32+
text: '3'
33+
Assign: [5, 0] - [5, 8]
34+
targets: [
35+
Name: [5, 0] - [5, 4]
36+
variable: Variable('\U00011db0', None)
37+
ctx: Store
38+
]
39+
value:
40+
Num: [5, 7] - [5, 8]
41+
n: 4
42+
text: '4'
43+
Assign: [8, 0] - [8, 8]
44+
targets: [
45+
Name: [8, 0] - [8, 4]
46+
variable: Variable('a\u1acf', None)
47+
ctx: Store
48+
]
49+
value:
50+
Num: [8, 7] - [8, 8]
51+
n: 5
52+
text: '5'
53+
Assign: [9, 0] - [9, 9]
54+
targets: [
55+
Name: [9, 0] - [9, 5]
56+
variable: Variable('a\U00011de0', None)
57+
ctx: Store
58+
]
59+
value:
60+
Num: [9, 8] - [9, 9]
61+
n: 6
62+
text: '6'
63+
Assign: [11, 0] - [11, 48]
64+
targets: [
65+
Name: [11, 0] - [11, 6]
66+
variable: Variable('result', None)
67+
ctx: Store
68+
]
69+
value:
70+
BinOp: [11, 9] - [11, 48]
71+
left:
72+
BinOp: [11, 9] - [11, 40]
73+
left:
74+
BinOp: [11, 9] - [11, 33]
75+
left:
76+
BinOp: [11, 9] - [11, 26]
77+
left:
78+
BinOp: [11, 9] - [11, 19]
79+
left:
80+
Name: [11, 9] - [11, 12]
81+
variable: Variable('\u088f', None)
82+
ctx: Load
83+
op: Add
84+
right:
85+
Name: [11, 15] - [11, 19]
86+
variable: Variable('\U0001e6c0', None)
87+
ctx: Load
88+
op: Add
89+
right:
90+
Name: [11, 22] - [11, 26]
91+
variable: Variable('\U000323b0', None)
92+
ctx: Load
93+
op: Add
94+
right:
95+
Name: [11, 29] - [11, 33]
96+
variable: Variable('\U00011db0', None)
97+
ctx: Load
98+
op: Add
99+
right:
100+
Name: [11, 36] - [11, 40]
101+
variable: Variable('a\u1acf', None)
102+
ctx: Load
103+
op: Add
104+
right:
105+
Name: [11, 43] - [11, 48]
106+
variable: Variable('a\U00011de0', None)
107+
ctx: Load
108+
]

‎python/extractor/tests/parser/unicode_identifiers_new.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Unicode 17 identifier starts: Arabic, Tai Yo, CJK Extension J, and Tolong Siki.
2+
࢏ = 1
3+
𞛀 = 2
4+
𲎰 = 3
5+
𑶰 = 4
6+
7+
# Unicode 17 identifier continuations: a combining mark and a decimal digit.
8+
a᫏ = 5
9+
a𑷠 = 6
10+
11+
result = ࢏ + 𞛀 + 𲎰 + 𑶰 + a᫏ + a𑷠

‎python/extractor/tsg-python/tsp/grammar.js‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,49 @@ const PREC = {
2222
call: 21,
2323
}
2424

25+
// The generator's Unicode properties currently use Unicode 16. Supplement them with
26+
// the Unicode 17 XID additions for Python 3.15. These can be removed once the generator
27+
// includes Unicode 17: https://www.unicode.org/Public/17.0.0/ucd/DerivedCoreProperties.txt
28+
const UNICODE17_XID_START = [
29+
String.raw`\u{88F}`,
30+
String.raw`\u{C5C}`,
31+
String.raw`\u{CDC}`,
32+
String.raw`\u{A7CE}-\u{A7CF}`,
33+
String.raw`\u{A7D2}`,
34+
String.raw`\u{A7D4}`,
35+
String.raw`\u{A7F1}`,
36+
String.raw`\u{10940}-\u{10959}`,
37+
String.raw`\u{10EC5}-\u{10EC7}`,
38+
String.raw`\u{11DB0}-\u{11DDB}`,
39+
String.raw`\u{16EA0}-\u{16EB8}`,
40+
String.raw`\u{16EBB}-\u{16ED3}`,
41+
String.raw`\u{16FF2}-\u{16FF6}`,
42+
String.raw`\u{187F8}-\u{187FF}`,
43+
String.raw`\u{18D09}-\u{18D1E}`,
44+
String.raw`\u{18D80}-\u{18DF2}`,
45+
String.raw`\u{1E6C0}-\u{1E6DE}`,
46+
String.raw`\u{1E6E0}-\u{1E6E2}`,
47+
String.raw`\u{1E6E4}-\u{1E6E5}`,
48+
String.raw`\u{1E6E7}-\u{1E6ED}`,
49+
String.raw`\u{1E6F0}-\u{1E6F4}`,
50+
String.raw`\u{1E6FE}-\u{1E6FF}`,
51+
String.raw`\u{2B73A}-\u{2B73F}`,
52+
String.raw`\u{2CEA2}-\u{2CEAD}`,
53+
String.raw`\u{323B0}-\u{33479}`,
54+
].join('');
55+
56+
const UNICODE17_XID_CONTINUE = UNICODE17_XID_START + [
57+
String.raw`\u{1ACF}-\u{1ADD}`,
58+
String.raw`\u{1AE0}-\u{1AEB}`,
59+
String.raw`\u{10EFA}-\u{10EFB}`,
60+
String.raw`\u{11B60}-\u{11B67}`,
61+
String.raw`\u{11DE0}-\u{11DE9}`,
62+
String.raw`\u{1E6E3}`,
63+
String.raw`\u{1E6E6}`,
64+
String.raw`\u{1E6EE}-\u{1E6EF}`,
65+
String.raw`\u{1E6F5}`,
66+
].join('');
67+
2568
module.exports = grammar({
2669
name: 'python',
2770

@@ -1220,7 +1263,10 @@ module.exports = grammar({
12201263
))
12211264
},
12221265

1223-
identifier: $ => /[_\p{XID_Start}][_\p{XID_Continue}]*/,
1266+
identifier: $ => new RegExp(
1267+
`[_\\p{XID_Start}${UNICODE17_XID_START}][_\\p{XID_Continue}${UNICODE17_XID_CONTINUE}]*`,
1268+
'u'
1269+
),
12241270

12251271
keyword_identifier: $ => prec(-3, alias(
12261272
choice(

‎python/ql/lib/change-notes/2026-09-22-python315-extractor.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@
22
category: minorAnalysis
33
---
44
* The Python extractor now supports unary plus in `match` literal patterns, introduced in Python 3.15.
5+
* The Python extractor now supports Unicode 17 identifiers, introduced in Python 3.15.

‎python/ql/test/3/extractor-tests/unicode17-identifiers/test.expected‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
| test.py:2:1:2:3 | \u088f | \u088f |
2+
| test.py:3:1:3:4 | \ufffd | \ufffd |
3+
| test.py:4:1:4:4 | \ufffd | \ufffd |
4+
| test.py:5:1:5:4 | \ufffd | \ufffd |
5+
| test.py:8:1:8:4 | a\u1acf | a\u1acf |
6+
| test.py:9:1:9:5 | a\ufffd | a\ufffd |
7+
| test.py:11:1:11:6 | result | result |
8+
| test.py:11:10:11:12 | \u088f | \u088f |
9+
| test.py:11:16:11:19 | \ufffd | \ufffd |
10+
| test.py:11:23:11:26 | \ufffd | \ufffd |
11+
| test.py:11:30:11:33 | \ufffd | \ufffd |
12+
| test.py:11:37:11:40 | a\u1acf | a\u1acf |
13+
| test.py:11:44:11:48 | a\ufffd | a\ufffd |

‎python/ql/test/3/extractor-tests/unicode17-identifiers/test.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Unicode 17 identifier starts: Arabic, Tai Yo, CJK Extension J, and Tolong Siki.
2+
࢏ = 1
3+
𞛀 = 2
4+
𲎰 = 3
5+
𑶰 = 4
6+
7+
# Unicode 17 identifier continuations: a combining mark and a decimal digit.
8+
a᫏ = 5
9+
a𑷠 = 6
10+
11+
result = ࢏ + 𞛀 + 𲎰 + 𑶰 + a᫏ + a𑷠

‎python/ql/test/3/extractor-tests/unicode17-identifiers/test.ql‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import python
2+
3+
from Name name
4+
where name.getLocation().getFile().getShortName() = "test.py"
5+
select name, name.getId()

0 commit comments

Comments
 (0)