Skip to content

Matchcompiler parse all Match|simpleMatch on the same line #2993

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
danmar merged 3 commits into from
Dec 31, 2020
Merged

Matchcompiler parse all Match|simpleMatch on the same line #2993

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c183dbb
MatchCompiler: Neaten error messages
rikardfalkeborn Dec 28, 2020
52c5b9f
MatchCompiler: Add spaces between operators
rikardfalkeborn Dec 20, 2020
5290f84
Matchcompiler: Don't bailout if non-const pattern
rikardfalkeborn Feb 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 49 additions & 47 deletions tools/matchcompiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def _compileCmd(tok):
elif tok == '%bool%':
return 'tok->isBoolean()'
elif tok == '%char%':
return '(tok->tokType()==Token::eChar)'
return '(tok->tokType() == Token::eChar)'
elif tok == '%comp%':
return 'tok->isComparisonOp()'
elif tok == '%num%':
Expand All @@ -102,24 +102,24 @@ def _compileCmd(tok):
elif tok == '%op%':
return 'tok->isOp()'
elif tok == '%or%':
return '(tok->tokType() == Token::eBitOp && tok->str()==MatchCompiler::makeConstString("|") )'
return '(tok->tokType() == Token::eBitOp && tok->str() == MatchCompiler::makeConstString("|") )'
elif tok == '%oror%':
return '(tok->tokType() == Token::eLogicalOp && tok->str()==MatchCompiler::makeConstString("||"))'
return '(tok->tokType() == Token::eLogicalOp && tok->str() == MatchCompiler::makeConstString("||"))'
elif tok == '%str%':
return '(tok->tokType()==Token::eString)'
return '(tok->tokType() == Token::eString)'
elif tok == '%type%':
return '(tok->isName() && tok->varId()==0U && (tok->str() != "delete" || !tok->isKeyword()))'
return '(tok->isName() && tok->varId() == 0U && (tok->str() != "delete" || !tok->isKeyword()))'
elif tok == '%name%':
return 'tok->isName()'
elif tok == '%var%':
return '(tok->varId() != 0)'
elif tok == '%varid%':
return '(tok->isName() && tok->varId()==varid)'
return '(tok->isName() && tok->varId() == varid)'
elif (len(tok) > 2) and (tok[0] == "%"):
print("unhandled:" + tok)

return (
'(tok->str()==MatchCompiler::makeConstString("' + tok + '"))'
'(tok->str() == MatchCompiler::makeConstString("' + tok + '"))'
)

def _compilePattern(self, pattern, nr, varid,
Expand Down Expand Up @@ -155,7 +155,7 @@ def _compilePattern(self, pattern, nr, varid,

# [abc]
if (len(tok) > 2) and (tok[0] == '[') and (tok[-1] == ']'):
ret += ' if (!tok || tok->str().size()!=1U || !strchr("' + tok[1:-1] + '", tok->str()[0]))\n'
ret += ' if (!tok || tok->str().size() != 1U || !strchr("' + tok[1:-1] + '", tok->str()[0]))\n'
ret += ' ' + returnStatement

# a|b|c
Expand Down Expand Up @@ -330,8 +330,8 @@ def _compileVerifyTokenMatch(
# ret += ' std::cout << "tok: " << tok->str();\n'
# ret += ' if (tok->next())\n'
# ret += ' std::cout << "tok next: " << tok->next()->str();\n'
ret += ' throw InternalError(tok, "Internal error.' +\
'compiled match returned different result than parsed match: ' + pattern + '");\n'
ret += ' throw InternalError(tok, "Internal error. ' +\
'Compiled match returned different result than parsed match: ' + pattern + '");\n'
ret += ' }\n'
ret += ' return res_compiled_match;\n'
ret += '}\n'
Expand Down Expand Up @@ -380,44 +380,46 @@ def _replaceSpecificTokenMatch(
)

def _replaceTokenMatch(self, line, linenr, filename):
while True:
is_simplematch = False
pos1 = line.find('Token::Match(')
if pos1 == -1:
is_simplematch = True
pos1 = line.find('Token::simpleMatch(')
if pos1 == -1:
break

res = self.parseMatch(line, pos1)
if res is None:
break

# assert that Token::Match has either 2 or 3 arguments
assert(len(res) == 3 or len(res) == 4)

end_pos = len(res[0])
tok = res[1]
raw_pattern = res[2]
varId = None
if len(res) == 4:
varId = res[3]
for func in ('Match', 'simpleMatch'):

@rikardfalkeborn rikardfalkeborn Dec 29, 2020

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was a bit back and forth if this should be a function which is called twice, instead of having a loop, but I guess it doesn't matter that much.

is_simplematch = func == 'simpleMatch'
pattern_start = 0
while True:
pos1 = line.find('Token::' + func + '(', pattern_start)
if pos1 == -1:
break

res = self.parseMatch(line, pos1)
if res is None:
break

# assert that Token::Match has either 2 or 3 arguments
assert(len(res) == 3 or len(res) == 4)

end_pos = len(res[0])
tok = res[1]
raw_pattern = res[2]
varId = None
if len(res) == 4:
varId = res[3]

res = re.match(r'\s*"((?:.|\\")*?)"\s*$', raw_pattern)
if res is None:
if self._showSkipped:
print(filename + ":" + str(linenr) + " skipping match pattern:" + raw_pattern)
break # Non-const pattern - bailout
pattern_start = pos1 + end_pos
res = re.match(r'\s*"((?:.|\\")*?)"\s*$', raw_pattern)
if res is None:
if self._showSkipped:
print(filename + ":" + str(linenr) + " skipping match pattern:" + raw_pattern)
continue # Non-const pattern - bailout

pattern = res.group(1)
line = self._replaceSpecificTokenMatch(
is_simplematch,
line,
pos1,
end_pos,
pattern,
tok,
varId)
pattern = res.group(1)
orig_len = len(line)
line = self._replaceSpecificTokenMatch(
is_simplematch,
line,
pos1,
end_pos,
pattern,
tok,
varId)
pattern_start += len(line) - orig_len

return line

Expand Down Expand Up @@ -459,7 +461,7 @@ def _compileVerifyTokenFindMatch(
# We also need to verify builds in 'release' mode
ret += ' if (res_parsed_findmatch != res_compiled_findmatch) {\n'
ret += ' throw InternalError(tok, "Internal error. ' +\
'compiled findmatch returned different result than parsed findmatch: ' + pattern + '");\n'
'Compiled findmatch returned different result than parsed findmatch: ' + pattern + '");\n'
ret += ' }\n'
ret += ' return res_compiled_findmatch;\n'
ret += '}\n'
Expand Down
23 changes: 23 additions & 0 deletions tools/test_matchcompiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ def test_replaceTokenMatch(self):
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
self.assertEqual(output, 'if (match1(tok)) {')

input = 'if (Token::simpleMatch(tok, "foobar")) {'
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
self.assertEqual(output, 'if (match1(tok)) {')

input = 'if (Token::Match(tok->next()->next(), "foobar %type% %num%")) {'
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
self.assertEqual(output, 'if (match2(tok->next()->next())) {')
Expand All @@ -60,6 +64,25 @@ def test_replaceTokenMatch(self):
self.assertEqual(
output, 'if (Token::Match(tok, "extern \"C\" " + varname)) {')

# test that multiple patterns on the same line are replaced
input = 'if (Token::Match(tok, "foo") && Token::Match(tok->next(), "baz")) {'
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
self.assertEqual(output, 'if (match4(tok) && match5(tok->next())) {')

# test that second pattern is replaced, even if there is a bailout on the first pattern
input = 'if (Token::Match(tok, foo) && Token::Match(tok->next(), "foobaz")) {'
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
self.assertEqual(output, 'if (Token::Match(tok, foo) && match6(tok->next())) {')

# test mixing Match and simpleMatch on the same line
input = 'if (Token::Match(tok, "a") && Token::simpleMatch(tok->next(), "b")) {'
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
self.assertEqual(output, 'if (match7(tok) && match8(tok->next())) {')

input = 'if (Token::simpleMatch(tok, "a") && Token::Match(tok->next(), "b")) {'
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
self.assertEqual(output, 'if (match7(tok) && match8(tok->next())) {')

def test_replaceTokenMatchWithVarId(self):
input = 'if (Token::Match(tok, "foobar %varid%", 123)) {'
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
Expand Down