Skip to content

Commit 4307d56

Browse files
andialbrechtandialbrecht
committed
Merge.
2 parents 1ad1131 + 5a1d58a commit 4307d56

3 files changed

Lines changed: 165 additions & 3 deletions

File tree

‎AUTHORS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,7 @@ python-sqlparse is written and maintained by Andi Albrecht <albrecht.andi@gmail.
33
This module contains code (namely the lexer and filter mechanism) from
44
the pygments project that was written by Georg Brandl.
55

6+
Alphabetical list of contributors:
7+
* Jesús Leganés Combarro
8+
9+

‎TODO‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
* See
2+
https://groups.google.com/d/msg/sqlparse/huz9lKXt0Lc/11ybIKPJWbUJ
3+
for some interesting hints and suggestions.
14
* Provide a function to replace tokens. See this thread: https://groups.google.com/d/msg/sqlparse/5xmBL2UKqX4/ZX9z_peve-AJ
25
* Fix bugs on issue tracker.
36
* Document filter stack and processing phases.

‎sqlparse/filters.py‎

Lines changed: 158 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,15 @@
22

33
import re
44

5-
from sqlparse import tokens as T
5+
from os.path import abspath, join
6+
67
from sqlparse import sql
8+
from sqlparse import tokens as T
9+
from sqlparse.engine import FilterStack
10+
from sqlparse.tokens import (
11+
Comment, Keyword, Name,
12+
Punctuation, String, Whitespace,
13+
)
714

815

916
class Filter(object):
@@ -52,6 +59,81 @@ def process(self, stack, stream):
5259
yield ttype, value
5360

5461

62+
class GetComments(Filter):
63+
"""Get the comments from a stack"""
64+
def process(self, stack, stream):
65+
for token_type, value in stream:
66+
if token_type in Comment:
67+
yield token_type, value
68+
69+
70+
class StripComments(Filter):
71+
"""Strip the comments from a stack"""
72+
def process(self, stack, stream):
73+
for token_type, value in stream:
74+
if token_type not in Comment:
75+
yield token_type, value
76+
77+
78+
class IncludeStatement(Filter):
79+
"""Filter that enable a INCLUDE statement"""
80+
81+
def __init__(self, dirpath=".", maxRecursive=10):
82+
self.dirpath = abspath(dirpath)
83+
self.maxRecursive = maxRecursive
84+
85+
self.detected = False
86+
87+
def process(self, stack, stream):
88+
# Run over all tokens in the stream
89+
for token_type, value in stream:
90+
# INCLUDE statement found, set detected mode
91+
if token_type in Name and value.upper() == 'INCLUDE':
92+
self.detected = True
93+
continue
94+
95+
# INCLUDE statement was found, parse it
96+
elif self.detected:
97+
# Omit whitespaces
98+
if token_type in Whitespace:
99+
pass
100+
101+
# Get path of file to include
102+
path = None
103+
104+
if token_type in String.Symbol:
105+
# if token_type in tokens.String.Symbol:
106+
path = join(self.dirpath, value[1:-1])
107+
108+
# Include file if path was found
109+
if path:
110+
try:
111+
f = open(path)
112+
raw_sql = f.read()
113+
f.close()
114+
except IOError, err:
115+
yield Comment, u'-- IOError: %s\n' % err
116+
117+
else:
118+
# Create new FilterStack to parse readed file
119+
# and add all its tokens to the main stack recursively
120+
# [ToDo] Add maximum recursive iteration value
121+
stack = FilterStack()
122+
stack.preprocess.append(IncludeStatement(self.dirpath))
123+
124+
for tv in stack.run(raw_sql):
125+
yield tv
126+
127+
# Set normal mode
128+
self.detected = False
129+
130+
# Don't include any token while in detected mode
131+
continue
132+
133+
# Normal token
134+
yield token_type, value
135+
136+
55137
# ----------------------
56138
# statement process
57139

@@ -146,13 +228,14 @@ def _split_kwds(self, tlist):
146228
split_words = ('FROM', 'JOIN$', 'AND', 'OR',
147229
'GROUP', 'ORDER', 'UNION', 'VALUES',
148230
'SET', 'BETWEEN')
231+
149232
def _next_token(i):
150233
t = tlist.token_next_match(i, T.Keyword, split_words,
151234
regex=True)
152235
if t and t.value.upper() == 'BETWEEN':
153-
t = _next_token(tlist.token_index(t)+1)
236+
t = _next_token(tlist.token_index(t) + 1)
154237
if t and t.value.upper() == 'AND':
155-
t = _next_token(tlist.token_index(t)+1)
238+
t = _next_token(tlist.token_index(t) + 1)
156239
return t
157240

158241
idx = 0
@@ -316,6 +399,57 @@ def process(self, stack, group):
316399
group.tokens = self._process(stack, group, group.tokens)
317400

318401

402+
class ColumnsSelect(Filter):
403+
"""Get the columns names of a SELECT query"""
404+
def process(self, stack, stream):
405+
mode = 0
406+
oldValue = ""
407+
parenthesis = 0
408+
409+
for token_type, value in stream:
410+
# Ignore comments
411+
if token_type in Comment:
412+
continue
413+
414+
# We have not detected a SELECT statement
415+
if mode == 0:
416+
if token_type in Keyword and value == 'SELECT':
417+
mode = 1
418+
419+
# We have detected a SELECT statement
420+
elif mode == 1:
421+
if value == 'FROM':
422+
if oldValue:
423+
yield Name, oldValue
424+
425+
mode = 3 # Columns have been checked
426+
427+
elif value == 'AS':
428+
oldValue = ""
429+
mode = 2
430+
431+
elif (token_type == Punctuation
432+
and value == ',' and not parenthesis):
433+
if oldValue:
434+
yield Name, oldValue
435+
oldValue = ""
436+
437+
elif token_type not in Whitespace:
438+
if value == '(':
439+
parenthesis += 1
440+
elif value == ')':
441+
parenthesis -= 1
442+
443+
oldValue += value
444+
445+
# We are processing an AS keyword
446+
elif mode == 2:
447+
# We check also for Keywords because a bug in SQLParse
448+
if token_type == Name or token_type == Keyword:
449+
yield Name, value
450+
mode = 1
451+
452+
319453
# ---------------------------
320454
# postprocess
321455

@@ -422,3 +556,24 @@ def process(self, stack, stmt):
422556
varname = self.varname
423557
stmt.tokens = tuple(self._process(stmt.tokens, varname))
424558
return stmt
559+
560+
561+
class Limit(Filter):
562+
"""Get the LIMIT of a query.
563+
564+
If not defined, return -1 (SQL specification for no LIMIT query)
565+
"""
566+
def process(self, stack, stream):
567+
index = 7
568+
stream = list(stream)
569+
stream.reverse()
570+
571+
# Run over all tokens in the stream from the end
572+
for token_type, value in stream:
573+
index -= 1
574+
575+
# if index and token_type in Keyword:
576+
if index and token_type in Keyword and value == 'LIMIT':
577+
return stream[4 - index][1]
578+
579+
return -1

0 commit comments

Comments
 (0)