Skip to content

Commit b0b3ca8

Browse files
arobenaroben
committed
Fix parsing of raw strings containing unclosed parentheses and brackets
Strings like r'[' and r'(' were causing the rest of the document to be parsed as if it were inside the regex. Now we terminate the regex at the closing quote in all cases. This is a backport of textmate/python.tmbundle@2567262
1 parent acd4ffa commit b0b3ca8

2 files changed

Lines changed: 186 additions & 86 deletions

File tree

‎grammars/python.cson‎

Lines changed: 42 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -994,36 +994,23 @@
994994
]
995995
}
996996
{
997-
'begin': '([uU]r)(")'
998-
'beginCaptures':
997+
'captures':
999998
'1':
1000999
'name': 'storage.type.string.python'
10011000
'2':
10021001
'name': 'punctuation.definition.string.begin.python'
1003-
'comment': 'double-quoted raw string'
1004-
'end': '((?<=")(")|")|(\\n)'
1005-
'endCaptures':
1006-
'1':
1007-
'name': 'punctuation.definition.string.end.python'
1008-
'2':
1009-
'name': 'meta.empty-string.double.python'
10101002
'3':
1011-
'name': 'invalid.illegal.unclosed-string.python'
1003+
'patterns': [
1004+
'include': '#constant_placeholder'
1005+
'include': '#escaped_unicode_char'
1006+
'include': '#escaped_char'
1007+
'include': '#regular_expressions'
1008+
]
1009+
'4':
1010+
'name': 'punctuation.definition.string.end.python'
1011+
'comment': 'double-quoted raw string'
1012+
'match': '([uU]r)(")((?:[^"\\\\]|\\\\.)*)(")'
10121013
'name': 'string.quoted.double.single-line.unicode-raw-regex.python'
1013-
'patterns': [
1014-
{
1015-
'include': '#constant_placeholder'
1016-
}
1017-
{
1018-
'include': '#escaped_unicode_char'
1019-
}
1020-
{
1021-
'include': '#escaped_char'
1022-
}
1023-
{
1024-
'include': '#regular_expressions'
1025-
}
1026-
]
10271014
}
10281015
{
10291016
'begin': '([uU]R)(")'
@@ -1055,33 +1042,22 @@
10551042
]
10561043
}
10571044
{
1058-
'begin': '(r)(")'
1059-
'beginCaptures':
1045+
'captures':
10601046
'1':
10611047
'name': 'storage.type.string.python'
10621048
'2':
10631049
'name': 'punctuation.definition.string.begin.python'
1064-
'comment': 'double-quoted raw string'
1065-
'end': '((?<=")(")|")|(\\n)'
1066-
'endCaptures':
1067-
'1':
1068-
'name': 'punctuation.definition.string.end.python'
1069-
'2':
1070-
'name': 'meta.empty-string.double.python'
10711050
'3':
1072-
'name': 'invalid.illegal.unclosed-string.python'
1051+
'patterns': [
1052+
'include': '#constant_placeholder'
1053+
'include': '#escaped_char'
1054+
'include': '#regular_expressions'
1055+
]
1056+
'4':
1057+
'name': 'punctuation.definition.string.end.python'
1058+
'comment': 'double-quoted raw string'
1059+
'match': '(r)(")((?:[^"\\\\]|\\\\.)*)(")'
10731060
'name': 'string.quoted.double.single-line.raw-regex.python'
1074-
'patterns': [
1075-
{
1076-
'include': '#constant_placeholder'
1077-
}
1078-
{
1079-
'include': '#escaped_char'
1080-
}
1081-
{
1082-
'include': '#regular_expressions'
1083-
}
1084-
]
10851061
}
10861062
{
10871063
'begin': '(R)(")'
@@ -1386,34 +1362,23 @@
13861362
]
13871363
}
13881364
{
1389-
'begin': '([uU]r)(\')'
1390-
'beginCaptures':
1365+
'captures':
13911366
'1':
13921367
'name': 'storage.type.string.python'
13931368
'2':
13941369
'name': 'punctuation.definition.string.begin.python'
1395-
'comment': 'single quoted raw string'
1396-
'end': '(\')|(\\n)'
1397-
'endCaptures':
1398-
'1':
1370+
'3':
1371+
'patterns': [
1372+
'include': '#constant_placeholder'
1373+
'include': '#escaped_unicode_char'
1374+
'include': '#escaped_char'
1375+
'include': '#regular_expressions'
1376+
]
1377+
'4':
13991378
'name': 'punctuation.definition.string.end.python'
1400-
'2':
1401-
'name': 'invalid.illegal.unclosed-string.python'
1379+
'comment': 'single quoted raw string'
1380+
'match': '([uU]r)(\')((?:[^\'\\\\]|\\\\.)*)(\')'
14021381
'name': 'string.quoted.single.single-line.unicode-raw-regex.python'
1403-
'patterns': [
1404-
{
1405-
'include': '#constant_placeholder'
1406-
}
1407-
{
1408-
'include': '#escaped_unicode_char'
1409-
}
1410-
{
1411-
'include': '#escaped_char'
1412-
}
1413-
{
1414-
'include': '#regular_expressions'
1415-
}
1416-
]
14171382
}
14181383
{
14191384
'begin': '([uU]R)(\')'
@@ -1443,31 +1408,22 @@
14431408
]
14441409
}
14451410
{
1446-
'begin': '(r)(\')'
1447-
'beginCaptures':
1411+
'captures':
14481412
'1':
14491413
'name': 'storage.type.string.python'
14501414
'2':
14511415
'name': 'punctuation.definition.string.begin.python'
1452-
'comment': 'single quoted raw string'
1453-
'end': '(\')|(\\n)'
1454-
'endCaptures':
1455-
'1':
1416+
'3':
1417+
'patterns': [
1418+
'include': '#constant_placeholder'
1419+
'include': '#escaped_char'
1420+
'include': '#regular_expressions'
1421+
]
1422+
'4':
14561423
'name': 'punctuation.definition.string.end.python'
1457-
'2':
1458-
'name': 'invalid.illegal.unclosed-string.python'
1424+
'comment': 'single quoted raw string'
1425+
'match': '(r)(\')((?:[^\'\\\\]|\\\\.)*)(\')'
14591426
'name': 'string.quoted.single.single-line.raw-regex.python'
1460-
'patterns': [
1461-
{
1462-
'include': '#constant_placeholder'
1463-
}
1464-
{
1465-
'include': '#escaped_char'
1466-
}
1467-
{
1468-
'include': '#regular_expressions'
1469-
}
1470-
]
14711427
}
14721428
{
14731429
'begin': '(R)(\')'

‎spec/python-spec.coffee‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,147 @@ describe "Python grammar", ->
3535
expect(tokens[1][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.python', 'punctuation.definition.string.end.python']
3636

3737
expect(tokens[1][2]).not.toBeDefined()
38+
39+
it "terminates a single-quoted raw string containing opening parenthesis at closing quote", ->
40+
tokens = grammar.tokenizeLines("r'(' #foo")
41+
42+
expect(tokens[0][0].value).toBe 'r'
43+
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'storage.type.string.python']
44+
expect(tokens[0][1].value).toBe "'"
45+
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.begin.python']
46+
expect(tokens[0][2].value).toBe '('
47+
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp']
48+
expect(tokens[0][3].value).toBe "'"
49+
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.end.python']
50+
expect(tokens[0][4].value).toBe ' '
51+
expect(tokens[0][4].scopes).toEqual ['source.python']
52+
expect(tokens[0][5].value).toBe '#'
53+
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
54+
expect(tokens[0][6].value).toBe 'foo'
55+
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']
56+
57+
it "terminates a single-quoted raw string containing opening bracket at closing quote", ->
58+
tokens = grammar.tokenizeLines("r'[' #foo")
59+
60+
expect(tokens[0][0].value).toBe 'r'
61+
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'storage.type.string.python']
62+
expect(tokens[0][1].value).toBe "'"
63+
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.begin.python']
64+
expect(tokens[0][2].value).toBe '['
65+
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp']
66+
expect(tokens[0][3].value).toBe "'"
67+
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.end.python']
68+
expect(tokens[0][4].value).toBe ' '
69+
expect(tokens[0][4].scopes).toEqual ['source.python']
70+
expect(tokens[0][5].value).toBe '#'
71+
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
72+
expect(tokens[0][6].value).toBe 'foo'
73+
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']
74+
75+
it "terminates a double-quoted raw string containing opening parenthesis at closing quote", ->
76+
tokens = grammar.tokenizeLines('r"(" #foo')
77+
78+
expect(tokens[0][0].value).toBe 'r'
79+
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'storage.type.string.python']
80+
expect(tokens[0][1].value).toBe '"'
81+
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.begin.python']
82+
expect(tokens[0][2].value).toBe '('
83+
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp']
84+
expect(tokens[0][3].value).toBe '"'
85+
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.end.python']
86+
expect(tokens[0][4].value).toBe ' '
87+
expect(tokens[0][4].scopes).toEqual ['source.python']
88+
expect(tokens[0][5].value).toBe '#'
89+
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
90+
expect(tokens[0][6].value).toBe 'foo'
91+
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']
92+
93+
it "terminates a double-quoted raw string containing opening bracket at closing quote", ->
94+
tokens = grammar.tokenizeLines('r"[" #foo')
95+
96+
expect(tokens[0][0].value).toBe 'r'
97+
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'storage.type.string.python']
98+
expect(tokens[0][1].value).toBe '"'
99+
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.begin.python']
100+
expect(tokens[0][2].value).toBe '['
101+
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp']
102+
expect(tokens[0][3].value).toBe '"'
103+
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.end.python']
104+
expect(tokens[0][4].value).toBe ' '
105+
expect(tokens[0][4].scopes).toEqual ['source.python']
106+
expect(tokens[0][5].value).toBe '#'
107+
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
108+
expect(tokens[0][6].value).toBe 'foo'
109+
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']
110+
111+
it "terminates a unicode single-quoted raw string containing opening parenthesis at closing quote", ->
112+
tokens = grammar.tokenizeLines("ur'(' #foo")
113+
114+
expect(tokens[0][0].value).toBe 'ur'
115+
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'storage.type.string.python']
116+
expect(tokens[0][1].value).toBe "'"
117+
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python']
118+
expect(tokens[0][2].value).toBe '('
119+
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp']
120+
expect(tokens[0][3].value).toBe "'"
121+
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python']
122+
expect(tokens[0][4].value).toBe ' '
123+
expect(tokens[0][4].scopes).toEqual ['source.python']
124+
expect(tokens[0][5].value).toBe '#'
125+
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
126+
expect(tokens[0][6].value).toBe 'foo'
127+
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']
128+
129+
it "terminates a unicode single-quoted raw string containing opening bracket at closing quote", ->
130+
tokens = grammar.tokenizeLines("ur'[' #foo")
131+
132+
expect(tokens[0][0].value).toBe 'ur'
133+
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'storage.type.string.python']
134+
expect(tokens[0][1].value).toBe "'"
135+
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python']
136+
expect(tokens[0][2].value).toBe '['
137+
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp']
138+
expect(tokens[0][3].value).toBe "'"
139+
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python']
140+
expect(tokens[0][4].value).toBe ' '
141+
expect(tokens[0][4].scopes).toEqual ['source.python']
142+
expect(tokens[0][5].value).toBe '#'
143+
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
144+
expect(tokens[0][6].value).toBe 'foo'
145+
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']
146+
147+
it "terminates a unicode double-quoted raw string containing opening parenthesis at closing quote", ->
148+
tokens = grammar.tokenizeLines('ur"(" #foo')
149+
150+
expect(tokens[0][0].value).toBe 'ur'
151+
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'storage.type.string.python']
152+
expect(tokens[0][1].value).toBe '"'
153+
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python']
154+
expect(tokens[0][2].value).toBe '('
155+
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp']
156+
expect(tokens[0][3].value).toBe '"'
157+
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python']
158+
expect(tokens[0][4].value).toBe ' '
159+
expect(tokens[0][4].scopes).toEqual ['source.python']
160+
expect(tokens[0][5].value).toBe '#'
161+
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
162+
expect(tokens[0][6].value).toBe 'foo'
163+
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']
164+
165+
it "terminates a unicode double-quoted raw string containing opening bracket at closing quote", ->
166+
tokens = grammar.tokenizeLines('ur"[" #foo')
167+
168+
expect(tokens[0][0].value).toBe 'ur'
169+
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'storage.type.string.python']
170+
expect(tokens[0][1].value).toBe '"'
171+
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python']
172+
expect(tokens[0][2].value).toBe '['
173+
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp']
174+
expect(tokens[0][3].value).toBe '"'
175+
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python']
176+
expect(tokens[0][4].value).toBe ' '
177+
expect(tokens[0][4].scopes).toEqual ['source.python']
178+
expect(tokens[0][5].value).toBe '#'
179+
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
180+
expect(tokens[0][6].value).toBe 'foo'
181+
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

0 commit comments

Comments
 (0)