Skip to content

Commit 8dc8aa0

Browse files
rikardfalkebornrikardfalkeborn
authored
Matchcompiler parse all Match|simpleMatch on the same line (#2993)
* MatchCompiler: Neaten error messages Especially the added space makes it a little more readable. * MatchCompiler: Add spaces between operators * Matchcompiler: Don't bailout if non-const pattern If matchcompiler found a call to Token::Match() or Token::simpleMatch() with an unknown string argument, subsequent calls to Token::Match() or Token::simpleMatch() on the same line would not be processed by matchcompiler. To fix this, keep track of the last index we found a match, and update it accordingly when the line is modified. To avoid having to keep track of if "Match" or "simpleMatch" is the first match we find, just make a loop over them.
1 parent e469436 commit 8dc8aa0

2 files changed

Lines changed: 72 additions & 47 deletions

File tree

‎tools/matchcompiler.py‎

Lines changed: 49 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ def _compileCmd(tok):
9292
elif tok == '%bool%':
9393
return 'tok->isBoolean()'
9494
elif tok == '%char%':
95-
return '(tok->tokType()==Token::eChar)'
95+
return '(tok->tokType() == Token::eChar)'
9696
elif tok == '%comp%':
9797
return 'tok->isComparisonOp()'
9898
elif tok == '%num%':
@@ -102,24 +102,24 @@ def _compileCmd(tok):
102102
elif tok == '%op%':
103103
return 'tok->isOp()'
104104
elif tok == '%or%':
105-
return '(tok->tokType() == Token::eBitOp && tok->str()==MatchCompiler::makeConstString("|") )'
105+
return '(tok->tokType() == Token::eBitOp && tok->str() == MatchCompiler::makeConstString("|") )'
106106
elif tok == '%oror%':
107-
return '(tok->tokType() == Token::eLogicalOp && tok->str()==MatchCompiler::makeConstString("||"))'
107+
return '(tok->tokType() == Token::eLogicalOp && tok->str() == MatchCompiler::makeConstString("||"))'
108108
elif tok == '%str%':
109-
return '(tok->tokType()==Token::eString)'
109+
return '(tok->tokType() == Token::eString)'
110110
elif tok == '%type%':
111-
return '(tok->isName() && tok->varId()==0U && (tok->str() != "delete" || !tok->isKeyword()))'
111+
return '(tok->isName() && tok->varId() == 0U && (tok->str() != "delete" || !tok->isKeyword()))'
112112
elif tok == '%name%':
113113
return 'tok->isName()'
114114
elif tok == '%var%':
115115
return '(tok->varId() != 0)'
116116
elif tok == '%varid%':
117-
return '(tok->isName() && tok->varId()==varid)'
117+
return '(tok->isName() && tok->varId() == varid)'
118118
elif (len(tok) > 2) and (tok[0] == "%"):
119119
print("unhandled:" + tok)
120120

121121
return (
122-
'(tok->str()==MatchCompiler::makeConstString("' + tok + '"))'
122+
'(tok->str() == MatchCompiler::makeConstString("' + tok + '"))'
123123
)
124124

125125
def _compilePattern(self, pattern, nr, varid,
@@ -155,7 +155,7 @@ def _compilePattern(self, pattern, nr, varid,
155155

156156
# [abc]
157157
if (len(tok) > 2) and (tok[0] == '[') and (tok[-1] == ']'):
158-
ret += ' if (!tok || tok->str().size()!=1U || !strchr("' + tok[1:-1] + '", tok->str()[0]))\n'
158+
ret += ' if (!tok || tok->str().size() != 1U || !strchr("' + tok[1:-1] + '", tok->str()[0]))\n'
159159
ret += ' ' + returnStatement
160160

161161
# a|b|c
@@ -330,8 +330,8 @@ def _compileVerifyTokenMatch(
330330
# ret += ' std::cout << "tok: " << tok->str();\n'
331331
# ret += ' if (tok->next())\n'
332332
# ret += ' std::cout << "tok next: " << tok->next()->str();\n'
333-
ret += ' throw InternalError(tok, "Internal error.' +\
334-
'compiled match returned different result than parsed match: ' + pattern + '");\n'
333+
ret += ' throw InternalError(tok, "Internal error. ' +\
334+
'Compiled match returned different result than parsed match: ' + pattern + '");\n'
335335
ret += ' }\n'
336336
ret += ' return res_compiled_match;\n'
337337
ret += '}\n'
@@ -380,44 +380,46 @@ def _replaceSpecificTokenMatch(
380380
)
381381

382382
def _replaceTokenMatch(self, line, linenr, filename):
383-
while True:
384-
is_simplematch = False
385-
pos1 = line.find('Token::Match(')
386-
if pos1 == -1:
387-
is_simplematch = True
388-
pos1 = line.find('Token::simpleMatch(')
389-
if pos1 == -1:
390-
break
391-
392-
res = self.parseMatch(line, pos1)
393-
if res is None:
394-
break
395-
396-
# assert that Token::Match has either 2 or 3 arguments
397-
assert(len(res) == 3 or len(res) == 4)
398-
399-
end_pos = len(res[0])
400-
tok = res[1]
401-
raw_pattern = res[2]
402-
varId = None
403-
if len(res) == 4:
404-
varId = res[3]
383+
for func in ('Match', 'simpleMatch'):
384+
is_simplematch = func == 'simpleMatch'
385+
pattern_start = 0
386+
while True:
387+
pos1 = line.find('Token::' + func + '(', pattern_start)
388+
if pos1 == -1:
389+
break
390+
391+
res = self.parseMatch(line, pos1)
392+
if res is None:
393+
break
394+
395+
# assert that Token::Match has either 2 or 3 arguments
396+
assert(len(res) == 3 or len(res) == 4)
397+
398+
end_pos = len(res[0])
399+
tok = res[1]
400+
raw_pattern = res[2]
401+
varId = None
402+
if len(res) == 4:
403+
varId = res[3]
405404

406-
res = re.match(r'\s*"((?:.|\\")*?)"\s*$', raw_pattern)
407-
if res is None:
408-
if self._showSkipped:
409-
print(filename + ":" + str(linenr) + " skipping match pattern:" + raw_pattern)
410-
break # Non-const pattern - bailout
405+
pattern_start = pos1 + end_pos
406+
res = re.match(r'\s*"((?:.|\\")*?)"\s*$', raw_pattern)
407+
if res is None:
408+
if self._showSkipped:
409+
print(filename + ":" + str(linenr) + " skipping match pattern:" + raw_pattern)
410+
continue # Non-const pattern - bailout
411411

412-
pattern = res.group(1)
413-
line = self._replaceSpecificTokenMatch(
414-
is_simplematch,
415-
line,
416-
pos1,
417-
end_pos,
418-
pattern,
419-
tok,
420-
varId)
412+
pattern = res.group(1)
413+
orig_len = len(line)
414+
line = self._replaceSpecificTokenMatch(
415+
is_simplematch,
416+
line,
417+
pos1,
418+
end_pos,
419+
pattern,
420+
tok,
421+
varId)
422+
pattern_start += len(line) - orig_len
421423

422424
return line
423425

@@ -459,7 +461,7 @@ def _compileVerifyTokenFindMatch(
459461
# We also need to verify builds in 'release' mode
460462
ret += ' if (res_parsed_findmatch != res_compiled_findmatch) {\n'
461463
ret += ' throw InternalError(tok, "Internal error. ' +\
462-
'compiled findmatch returned different result than parsed findmatch: ' + pattern + '");\n'
464+
'Compiled findmatch returned different result than parsed findmatch: ' + pattern + '");\n'
463465
ret += ' }\n'
464466
ret += ' return res_compiled_findmatch;\n'
465467
ret += '}\n'

‎tools/test_matchcompiler.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ def test_replaceTokenMatch(self):
3939
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
4040
self.assertEqual(output, 'if (match1(tok)) {')
4141

42+
input = 'if (Token::simpleMatch(tok, "foobar")) {'
43+
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
44+
self.assertEqual(output, 'if (match1(tok)) {')
45+
4246
input = 'if (Token::Match(tok->next()->next(), "foobar %type% %num%")) {'
4347
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
4448
self.assertEqual(output, 'if (match2(tok->next()->next())) {')
@@ -60,6 +64,25 @@ def test_replaceTokenMatch(self):
6064
self.assertEqual(
6165
output, 'if (Token::Match(tok, "extern \"C\" " + varname)) {')
6266

67+
# test that multiple patterns on the same line are replaced
68+
input = 'if (Token::Match(tok, "foo") && Token::Match(tok->next(), "baz")) {'
69+
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
70+
self.assertEqual(output, 'if (match4(tok) && match5(tok->next())) {')
71+
72+
# test that second pattern is replaced, even if there is a bailout on the first pattern
73+
input = 'if (Token::Match(tok, foo) && Token::Match(tok->next(), "foobaz")) {'
74+
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
75+
self.assertEqual(output, 'if (Token::Match(tok, foo) && match6(tok->next())) {')
76+
77+
# test mixing Match and simpleMatch on the same line
78+
input = 'if (Token::Match(tok, "a") && Token::simpleMatch(tok->next(), "b")) {'
79+
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
80+
self.assertEqual(output, 'if (match7(tok) && match8(tok->next())) {')
81+
82+
input = 'if (Token::simpleMatch(tok, "a") && Token::Match(tok->next(), "b")) {'
83+
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
84+
self.assertEqual(output, 'if (match7(tok) && match8(tok->next())) {')
85+
6386
def test_replaceTokenMatchWithVarId(self):
6487
input = 'if (Token::Match(tok, "foobar %varid%", 123)) {'
6588
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")

0 commit comments

Comments
 (0)