Skip to content

Commit 9a1cb5d

Browse files
mrmasterplanandialbrecht
mrmasterplan
authored and
andialbrecht
committed
configurable syntax
1 parent 8b789f2 commit 9a1cb5d

3 files changed

Lines changed: 82 additions & 30 deletions

File tree

‎sqlparse/keywords.py‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,23 +6,17 @@
66
# the BSD License: https://opensource.org/licenses/BSD-3-Clause
77

88
import re
9+
from typing import Dict, List, Tuple, Callable, Union
910

1011
from sqlparse import tokens
1112

13+
# object() only supports "is" and is useful as a marker
14+
PROCESS_AS_KEYWORD = object()
1215

13-
def is_keyword(value):
14-
"""Checks for a keyword.
15-
16-
If the given value is in one of the KEYWORDS_* dictionary
17-
it's considered a keyword. Otherwise tokens.Name is returned.
18-
"""
19-
val = value.upper()
20-
return (KEYWORDS_COMMON.get(val)
21-
or KEYWORDS_ORACLE.get(val)
22-
or KEYWORDS_PLPGSQL.get(val)
23-
or KEYWORDS_HQL.get(val)
24-
or KEYWORDS_MSACCESS.get(val)
25-
or KEYWORDS.get(val, tokens.Name)), value
16+
SQL_REGEX_TYPE = List[
17+
Tuple[Callable, Union[type(PROCESS_AS_KEYWORD), tokens._TokenType]]
18+
]
19+
KEYWORDS_TYPE = Dict[str, tokens._TokenType]
2620

2721

2822
SQL_REGEX = {
@@ -99,7 +93,7 @@ def is_keyword(value):
9993
(r'(NOT\s+)?(REGEXP)\b', tokens.Operator.Comparison),
10094
# Check for keywords, also returns tokens.Name if regex matches
10195
# but the match isn't a keyword.
102-
(r'[0-9_\w][_$#\w]*', is_keyword),
96+
(r'[0-9_\w][_$#\w]*', PROCESS_AS_KEYWORD),
10397
(r'[;:()\[\],\.]', tokens.Punctuation),
10498
(r'[<>=~!]+', tokens.Operator.Comparison),
10599
(r'[+/@#%^&|^-]+', tokens.Operator),

‎sqlparse/lexer.py‎

Lines changed: 72 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,74 @@
1313
# and to allow some customizations.
1414

1515
from io import TextIOBase
16+
from typing import List
1617

17-
from sqlparse import tokens
18-
from sqlparse.keywords import SQL_REGEX
18+
from sqlparse import tokens, keywords
1919
from sqlparse.utils import consume
2020

2121

22-
class Lexer:
23-
"""Lexer
24-
Empty class. Leaving for backwards-compatibility
25-
"""
22+
class _LexerSingletonMetaclass(type):
23+
_lexer_instance = None
24+
25+
def __call__(cls, *args, **kwargs):
26+
if _LexerSingletonMetaclass._lexer_instance is None:
27+
_LexerSingletonMetaclass._lexer_instance = super(
28+
_LexerSingletonMetaclass, cls
29+
).__call__(*args, **kwargs)
30+
return _LexerSingletonMetaclass._lexer_instance
31+
32+
33+
class Lexer(metaclass=_LexerSingletonMetaclass):
34+
"""The Lexer supports configurable syntax.
35+
To add support for additional keywords, use the `add_keywords` method."""
36+
37+
_SQL_REGEX: keywords.SQL_REGEX_TYPE
38+
_keywords: List[keywords.KEYWORDS_TYPE]
39+
40+
def default_initialization(self):
41+
"""Initialize the lexer with default dictionaries.
42+
Useful if you need to revert custom syntax settings."""
43+
self.clear()
44+
self.set_SQL_REGEX(keywords.SQL_REGEX)
45+
self.add_keywords(keywords.KEYWORDS_COMMON)
46+
self.add_keywords(keywords.KEYWORDS_ORACLE)
47+
self.add_keywords(keywords.KEYWORDS_PLPGSQL)
48+
self.add_keywords(keywords.KEYWORDS_HQL)
49+
self.add_keywords(keywords.KEYWORDS_MSACCESS)
50+
self.add_keywords(keywords.KEYWORDS)
51+
52+
def __init__(self):
53+
self.default_initialization()
54+
55+
def clear(self):
56+
"""Clear all syntax configurations.
57+
Useful if you want to load a reduced set of syntax configurations."""
58+
self._SQL_REGEX = []
59+
self._keywords = []
60+
61+
def set_SQL_REGEX(self, SQL_REGEX: keywords.SQL_REGEX_TYPE):
62+
"""Set the list of regex that will parse the SQL."""
63+
self._SQL_REGEX = SQL_REGEX
64+
65+
def add_keywords(self, keywords: keywords.KEYWORDS_TYPE):
66+
"""Add keyword dictionaries. Keywords are looked up in the same order
67+
that dictionaries were added."""
68+
self._keywords.append(keywords)
69+
70+
def is_keyword(self, value):
71+
"""Checks for a keyword.
72+
73+
If the given value is in one of the KEYWORDS_* dictionary
74+
it's considered a keyword. Otherwise tokens.Name is returned.
75+
"""
76+
val = value.upper()
77+
for kwdict in self._keywords:
78+
if val in kwdict:
79+
return kwdict[val], value
80+
else:
81+
return tokens.Name, value
2682

27-
@staticmethod
28-
def get_tokens(text, encoding=None):
83+
def get_tokens(self, text, encoding=None):
2984
"""
3085
Return an iterable of (tokentype, value) pairs generated from
3186
`text`. If `unfiltered` is set to `True`, the filtering mechanism
@@ -48,24 +103,26 @@ def get_tokens(text, encoding=None):
48103
text = text.decode(encoding)
49104
else:
50105
try:
51-
text = text.decode('utf-8')
106+
text = text.decode("utf-8")
52107
except UnicodeDecodeError:
53-
text = text.decode('unicode-escape')
108+
text = text.decode("unicode-escape")
54109
else:
55-
raise TypeError("Expected text or file-like object, got {!r}".
56-
format(type(text)))
110+
raise TypeError(
111+
"Expected text or file-like object, got {!r}"
112+
.format(type(text))
113+
)
57114

58115
iterable = enumerate(text)
59116
for pos, char in iterable:
60-
for rexmatch, action in SQL_REGEX:
117+
for rexmatch, action in self._SQL_REGEX:
61118
m = rexmatch(text, pos)
62119

63120
if not m:
64121
continue
65122
elif isinstance(action, tokens._TokenType):
66123
yield action, m.group()
67-
elif callable(action):
68-
yield action(m.group())
124+
elif action is keywords.PROCESS_AS_KEYWORD:
125+
yield self.is_keyword(m.group())
69126

70127
consume(iterable, m.end() - pos - 1)
71128
break

‎tests/test_keywords.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,13 @@
22

33
from sqlparse import tokens
44
from sqlparse.keywords import SQL_REGEX
5+
from sqlparse.lexer import Lexer
56

67

78
class TestSQLREGEX:
89
@pytest.mark.parametrize('number', ['1.0', '-1.0',
910
'1.', '-1.',
1011
'.1', '-.1'])
1112
def test_float_numbers(self, number):
12-
ttype = next(tt for action, tt in SQL_REGEX if action(number))
13+
ttype = next(tt for action, tt in Lexer()._SQL_REGEX if action(number))
1314
assert tokens.Number.Float == ttype

0 commit comments

Comments
 (0)