# a glorified C pre-processor parser

import sys, re, string

from utils import *

from defaults import *

debugTokens = False

debugDirectiveTokenizer = False

debugLineParsing = False

debugCppExpr = False

debugOptimIf01 = False

#####################################################################################

#####################################################################################

##### #####

##### C P P T O K E N S #####

##### #####

#####################################################################################

#####################################################################################

# the list of supported C-preprocessor tokens

# plus a couple of C tokens as well

tokEOF = "\0"

tokLN = "\n"

tokSTRINGIFY = "#"

tokCONCAT = "##"

tokLOGICAND = "&&"

tokLOGICOR = "||"

tokSHL = "<<"

tokSHR = ">>"

tokEQUAL = "=="

tokNEQUAL = "!="

tokLT = "<"

tokLTE = "<="

tokGT = ">"

tokGTE = ">="

tokELLIPSIS = "..."

tokSPACE = " "

tokDEFINED = "defined"

tokLPAREN = "("

tokRPAREN = ")"

tokNOT = "!"

tokPLUS = "+"

tokMINUS = "-"

tokMULTIPLY = "*"

tokDIVIDE = "/"

tokMODULUS = "%"

tokBINAND = "&"

tokBINOR = "|"

tokBINXOR = "^"

tokCOMMA = ","

tokLBRACE = "{"

tokRBRACE = "}"

tokARROW = "->"

tokINCREMENT = "++"

tokDECREMENT = "--"

tokNUMBER = "<number>"

tokIDENT = "<ident>"

tokSTRING = "<string>"

class Token:

"""a simple class to hold information about a given token.

each token has a position in the source code, as well as

an 'id' and a 'value'. the id is a string that identifies

the token's class, while the value is the string of the

original token itself.

for example, the tokenizer concatenates a series of spaces

and tabs as a single tokSPACE id, whose value if the original

spaces+tabs sequence."""

def __init__(self):

self.id = None

self.value = None

self.lineno = 0

self.colno = 0

def set(self,id,val=None):

self.id = id

if val:

self.value = val

else:

self.value = id

return None

def copyFrom(self,src):

self.id = src.id

self.value = src.value

self.lineno = src.lineno

self.colno = src.colno

def __repr__(self):

if self.id == tokIDENT:

return "(ident %s)" % self.value

if self.id == tokNUMBER:

return "(number %s)" % self.value

if self.id == tokSTRING:

return "(string '%s')" % self.value

if self.id == tokLN:

return "<LN>"

if self.id == tokEOF:

return "<EOF>"

if self.id == tokSPACE and self.value == "\\":

# this corresponds to a trailing \ that was transformed into a tokSPACE

return "<\\>"

return self.id

def __str__(self):

if self.id == tokIDENT:

return self.value

if self.id == tokNUMBER:

return self.value

if self.id == tokSTRING:

return self.value

if self.id == tokEOF:

return "<EOF>"

if self.id == tokSPACE:

if self.value == "\\": # trailing \

return "\\\n"

else:

return self.value

return self.id

class BadExpectedToken(Exception):

def __init__(self,msg):

print msg

#####################################################################################

#####################################################################################

##### #####

##### C P P T O K E N C U R S O R #####

##### #####

#####################################################################################

#####################################################################################

class TokenCursor:

"""a small class to iterate over a list of Token objects"""

def __init__(self,tokens):

self.tokens = tokens

self.n = 0

self.count = len(tokens)

def set(self,n):

"""set the current position"""

if n < 0:

n = 0

if n > self.count:

n = self.count

self.n = n

def peekId(self):

"""retrieve the id of the current token"""

if (self.n >= self.count):

return None

return self.tokens[self.n].id

def peek(self):

"""retrieve the current token. does not change position"""

if (self.n >= self.count):

return None

return self.tokens[self.n]

def skip(self):

"""increase current token position"""

if (self.n < self.count):

self.n += 1

def skipSpaces(self):

"""skip over all space tokens, this includes tokSPACE and tokLN"""

while 1:

tok = self.peekId()

if tok != tokSPACE and tok != tokLN:

break

self.skip()

def skipIfId(self,id):

"""skip an optional token"""

if self.peekId() == id:

self.skip()

def expectId(self,id):

"""raise an exception if the current token hasn't a given id.

otherwise skip over it"""

tok = self.peek()

if tok.id != id:

raise BadExpectedToken, "%d:%d: '%s' expected, received '%s'" % (tok.lineno, tok.colno, id, tok.id)

self.skip()

def remain(self):

"""return the list of remaining tokens"""

return self.tokens[self.n:]

#####################################################################################

#####################################################################################

##### #####

##### C P P T O K E N I Z E R #####

##### #####

#####################################################################################

#####################################################################################

# list of long symbols, i.e. those that take more than one characters

cppLongSymbols = [ tokCONCAT, tokLOGICAND, tokLOGICOR, tokSHL, tokSHR, tokELLIPSIS, tokEQUAL,\

tokNEQUAL, tokLTE, tokGTE, tokARROW, tokINCREMENT, tokDECREMENT ]

class CppTokenizer:

"""an abstract class used to convert some input text into a list

of tokens. real implementations follow and differ in the format

of the input text only"""

def __init__(self):

"""initialize a new CppTokenizer object"""

self.eof = False # end of file reached ?

self.text = None # content of current line, with final \n stripped

self.line = 0 # number of current line

self.pos = 0 # current character position in current line

self.len = 0 # length of current line text

self.held = Token()

def setLineText(self,line):

"""set the content of the (next) current line. should be called

by fillLineText() in derived classes"""

self.text = line

self.len = len(line)

self.pos = 0

def fillLineText(self):

"""refresh the content of 'line' with a new line of input"""

# to be overriden

self.eof = True

def markPos(self,tok):

"""mark the position of the current token in the source file"""

if self.eof or self.pos > self.len:

tok.lineno = self.line + 1

tok.colno = 0

else:

tok.lineno = self.line

tok.colno = self.pos

def peekChar(self):

"""return the current token under the cursor without moving it"""

if self.eof:

return tokEOF

if self.pos > self.len:

self.pos = 0

self.line += 1

self.fillLineText()

if self.eof:

return tokEOF

if self.pos == self.len:

return tokLN

else:

return self.text[self.pos]

def peekNChar(self,n):

"""try to peek the next n chars on the same line"""

if self.pos + n > self.len:

return None

return self.text[self.pos:self.pos+n]

def skipChar(self):

"""increment the token cursor position"""

if not self.eof:

self.pos += 1

def skipNChars(self,n):

if self.pos + n <= self.len:

self.pos += n

else:

while n > 0:

self.skipChar()

n -= 1

def nextChar(self):

"""retrieve the token at the current cursor position, then skip it"""

result = self.peekChar()

self.skipChar()

return result

def getEscape(self):

# try to get all characters after a backslash (\)

result = self.nextChar()

if result == "0":

# octal number ?

num = self.peekNChar(3)

if num != None:

isOctal = True

for d in num:

if not d in "01234567":

isOctal = False

break

if isOctal:

result += num

self.skipNChars(3)

elif result == "x" or result == "X":

# hex number ?

num = self.peekNChar(2)

if num != None:

isHex = True

for d in num:

if not d in "012345678abcdefABCDEF":

isHex = False

break

if isHex:

result += num

self.skipNChars(2)

elif result == "u" or result == "U":

# unicode char ?

num = self.peekNChar(4)

if num != None:

isHex = True

for d in num:

if not d in "012345678abcdefABCDEF":

isHex = False

break

if isHex:

result += num

self.skipNChars(4)

return result

def nextRealToken(self,tok):

"""return next CPP token, used internally by nextToken()"""

c = self.nextChar()

if c == tokEOF or c == tokLN:

return tok.set(c)

if c == '/':

c = self.peekChar()

if c == '/': # C++ comment line

self.skipChar()

while 1:

c = self.nextChar()

if c == tokEOF or c == tokLN:

break

return tok.set(tokLN)

if c == '*': # C comment start

self.skipChar()

value = "/*"

prev_c = None

while 1:

c = self.nextChar()

if c == tokEOF:

#print "## EOF after '%s'" % value

return tok.set(tokEOF,value)

if c == '/' and prev_c == '*':

break

prev_c = c

value += c

value += "/"

#print "## COMMENT: '%s'" % value

return tok.set(tokSPACE,value)

c = '/'

if c.isspace():

while 1:

c2 = self.peekChar()

if c2 == tokLN or not c2.isspace():

break

c += c2

self.skipChar()

return tok.set(tokSPACE,c)

if c == '\\':

if debugTokens:

print "nextRealToken: \\ found, next token is '%s'" % repr(self.peekChar())

if self.peekChar() == tokLN: # trailing \

# eat the tokLN

self.skipChar()

# we replace a trailing \ by a tokSPACE whose value is

# simply "\\". this allows us to detect them later when

# needed.

return tok.set(tokSPACE,"\\")

else:

# treat as a single token here ?

c +=self.getEscape()

return tok.set(c)

if c == "'": # chars

c2 = self.nextChar()

c += c2

if c2 == '\\':

c += self.getEscape()

while 1:

c2 = self.nextChar()

if c2 == tokEOF:

break

c += c2

if c2 == "'":

break

return tok.set(tokSTRING, c)

if c == '"': # strings

quote = 0

while 1:

c2 = self.nextChar()

if c2 == tokEOF:

return tok.set(tokSTRING,c)

c += c2

if not quote:

if c2 == '"':

return tok.set(tokSTRING,c)

if c2 == "\\":

quote = 1

else:

quote = 0

if c >= "0" and c <= "9": # integers ?

while 1:

c2 = self.peekChar()

if c2 == tokLN or (not c2.isalnum() and c2 != "_"):

break

c += c2

self.skipChar()

return tok.set(tokNUMBER,c)

if c.isalnum() or c == "_": # identifiers ?

while 1:

c2 = self.peekChar()

if c2 == tokLN or (not c2.isalnum() and c2 != "_"):

break

c += c2

self.skipChar()

if c == tokDEFINED:

return tok.set(tokDEFINED)

else:

return tok.set(tokIDENT,c)

# check special symbols

for sk in cppLongSymbols:

if c == sk[0]:

sklen = len(sk[1:])

if self.pos + sklen <= self.len and \

self.text[self.pos:self.pos+sklen] == sk[1:]:

self.pos += sklen

return tok.set(sk)

return tok.set(c)

def nextToken(self,tok):

"""return the next token from the input text. this function

really updates 'tok', and does not return a new one"""

self.markPos(tok)

self.nextRealToken(tok)

def getToken(self):

tok = Token()

self.nextToken(tok)

if debugTokens:

print "getTokens: %s" % repr(tok)

return tok

def toTokenList(self):

"""convert the input text of a CppTokenizer into a direct

list of token objects. tokEOF is stripped from the result"""

result = []

while 1:

tok = Token()

self.nextToken(tok)

if tok.id == tokEOF:

break

result.append(tok)

return result

class CppLineTokenizer(CppTokenizer):

"""a CppTokenizer derived class that accepts a single line of text as input"""

def __init__(self,line,lineno=1):

CppTokenizer.__init__(self)

self.line = lineno

self.setLineText(line)

class CppLinesTokenizer(CppTokenizer):

"""a CppTokenizer derived class that accepts a list of texdt lines as input.

the lines must not have a trailing \n"""

def __init__(self,lines=[],lineno=1):

"""initialize a CppLinesTokenizer. you can later add lines using addLines()"""

CppTokenizer.__init__(self)

self.line = lineno

self.lines = lines

self.index = 0

self.count = len(lines)

if self.count > 0:

self.fillLineText()

else:

self.eof = True

def addLine(self,line):

"""add a line to a CppLinesTokenizer. this can be done after tokenization

happens"""

if self.count == 0:

self.setLineText(line)

self.index = 1

self.lines.append(line)

self.count += 1

self.eof = False

def fillLineText(self):

if self.index < self.count:

self.setLineText(self.lines[self.index])

self.index += 1

else:

self.eof = True

class CppFileTokenizer(CppTokenizer):

def __init__(self,file,lineno=1):

CppTokenizer.__init__(self)

self.file = file

self.line = lineno

def fillLineText(self):

line = self.file.readline()

if len(line) > 0:

if line[-1] == '\n':

line = line[:-1]

if len(line) > 0 and line[-1] == "\r":

line = line[:-1]

self.setLineText(line)

else:

self.eof = True

# Unit testing

#

class CppTokenizerTester:

"""a class used to test CppTokenizer classes"""

def __init__(self,tokenizer=None):

self.tokenizer = tokenizer

self.token = Token()

def setTokenizer(self,tokenizer):

self.tokenizer = tokenizer

def expect(self,id):

self.tokenizer.nextToken(self.token)

tokid = self.token.id

if tokid == id:

return

if self.token.value == id and (tokid == tokIDENT or tokid == tokNUMBER):

return

raise BadExpectedToken, "### BAD TOKEN: '%s' expecting '%s'" % (self.token.id,id)

def expectToken(self,id,line,col):

self.expect(id)

if self.token.lineno != line:

raise BadExpectedToken, "### BAD LINENO: token '%s' got '%d' expecting '%d'" % (id,self.token.lineno,line)

if self.token.colno != col:

raise BadExpectedToken, "### BAD COLNO: '%d' expecting '%d'" % (self.token.colno,col)

def expectTokenVal(self,id,value,line,col):

self.expectToken(id,line,col)

if self.token.value != value:

raise BadExpectedToken, "### BAD VALUE: '%s' expecting '%s'" % (self.token.value,value)

def expectList(self,list):

for item in list:

self.expect(item)

def test_CppTokenizer():

print "running CppTokenizer tests"

tester = CppTokenizerTester()

tester.setTokenizer( CppLineTokenizer("#an/example && (01923_xy)") )

tester.expectList( ["#", "an", "/", "example", tokSPACE, tokLOGICAND, tokSPACE, tokLPAREN, "01923_xy", \

tokRPAREN, tokLN, tokEOF] )

tester.setTokenizer( CppLineTokenizer("FOO(BAR) && defined(BAZ)") )

tester.expectList( ["FOO", tokLPAREN, "BAR", tokRPAREN, tokSPACE, tokLOGICAND, tokSPACE,

tokDEFINED, tokLPAREN, "BAZ", tokRPAREN, tokLN, tokEOF] )

tester.setTokenizer( CppLinesTokenizer( ["/*", "#", "*/"] ) )

tester.expectList( [ tokSPACE, tokLN, tokEOF ] )

tester.setTokenizer( CppLinesTokenizer( ["first", "second"] ) )

tester.expectList( [ "first", tokLN, "second", tokLN, tokEOF ] )

tester.setTokenizer( CppLinesTokenizer( ["first second", " third"] ) )

tester.expectToken( "first", 1, 0 )

tester.expectToken( tokSPACE, 1, 5 )

tester.expectToken( "second", 1, 6 )

tester.expectToken( tokLN, 1, 12 )

tester.expectToken( tokSPACE, 2, 0 )

tester.expectToken( "third", 2, 2 )

tester.setTokenizer( CppLinesTokenizer( [ "boo /* what the", "hell */" ] ) )

tester.expectList( [ "boo", tokSPACE ] )

tester.expectTokenVal( tokSPACE, "/* what the\nhell */", 1, 4 )

tester.expectList( [ tokLN, tokEOF ] )

tester.setTokenizer( CppLinesTokenizer( [ "an \\", " example" ] ) )

tester.expectToken( "an", 1, 0 )

tester.expectToken( tokSPACE, 1, 2 )

tester.expectTokenVal( tokSPACE, "\\", 1, 3 )

tester.expectToken( tokSPACE, 2, 0 )

tester.expectToken( "example", 2, 1 )

tester.expectToken( tokLN, 2, 8 )

return True

#####################################################################################

#####################################################################################

##### #####

##### C P P E X P R E S S I O N S #####

##### #####

#####################################################################################

#####################################################################################

# Cpp expressions are modeled by tuples of the form (op,arg) or (op,arg1,arg2), etc..

# op is an "operator" string

class Expr:

"""a class used to model a CPP expression"""

opInteger = "int"

opIdent = "ident"

opCall = "call"

opDefined = "defined"

opTest = "?"

opLogicNot = "!"

opNot = "~"

opNeg = "[-]"

opUnaryPlus = "[+]"

opAdd = "+"

opSub = "-"

opMul = "*"

opDiv = "/"

opMod = "%"

opAnd = "&"

opOr = "|"

opXor = "^"

opLogicAnd = "&&"

opLogicOr = "||"

opEqual = "=="

opNotEqual = "!="

opLess = "<"

opLessEq = "<="

opGreater = ">"

opGreaterEq = ">="

opShl = "<<"

opShr = ">>"

unaries = [ opLogicNot, opNot, opNeg, opUnaryPlus ]

binaries = [ opAdd, opSub, opMul, opDiv, opMod, opAnd, opOr, opXor, opLogicAnd, opLogicOr,

opEqual, opNotEqual, opLess, opLessEq, opGreater, opGreaterEq ]

precedences = {

opTest: 0,

opLogicOr: 1,

opLogicNot: 2,

opOr : 3,

opXor: 4,

opAnd: 5,

opEqual: 6, opNotEqual: 6,

opLess:7, opLessEq:7, opGreater:7, opGreaterEq:7,

opShl:8, opShr:8,

opAdd:9, opSub:9,

opMul:10, opDiv:10, opMod:10,

opLogicNot:11,

opNot: 12,

}

def __init__(self,op):

self.op = op

def __repr__(self):

return "(%s)" % self.op

def __str__(self):

return "operator(%s)" % self.op

def precedence(self):

"""return the precedence of a given operator"""

return Expr.precedences.get(self.op, 1000)

def isUnary(self):

return self.op in Expr.unaries

def isBinary(self):

return self.op in Expr.binaries

def isDefined(self):

return self.op is opDefined

def toInt(self):

"""return the integer value of a given expression. only valid for integer expressions

will return None otherwise"""

return None

class IntExpr(Expr):

def __init__(self,value):

Expr.__init__(self,opInteger)

self.arg = value

def __repr__(self):

return "(int %s)" % self.arg

def __str__(self):

return self.arg

def toInt(self):

s = self.arg # string value

# get rid of U or L suffixes

while len(s) > 0 and s[-1] in "LUlu":

s = s[:-1]

return string.atoi(s)

class IdentExpr(Expr):

def __init__(self,name):

Expr.__init__(self,opIdent)

self.name = name

def __repr__(self):

return "(ident %s)" % self.name

def __str__(self):

return self.name

class CallExpr(Expr):

def __init__(self,funcname,params):

Expr.__init__(self,opCall)

self.funcname = funcname

self.params = params

def __repr__(self):

result = "(call %s [" % self.funcname

comma = ""

for param in self.params:

result += "%s%s" % (comma, repr(param))

comma = ","

result += "])"

return result

def __str__(self):

result = "%s(" % self.funcname

comma = ""

for param in self.params:

result += "%s%s" % (comma, str(param))

comma = ","

result += ")"

return result

class TestExpr(Expr):

def __init__(self,cond,iftrue,iffalse):

Expr.__init__(self,opTest)

self.cond = cond

self.iftrue = iftrue

self.iffalse = iffalse

def __repr__(self):

return "(?: %s %s %s)" % (repr(self.cond),repr(self.iftrue),repr(self.iffalse))

def __str__(self):

return "(%s) ? (%s) : (%s)" % (self.cond, self.iftrue, self.iffalse)

class SingleArgExpr(Expr):

def __init__(self,op,arg):

Expr.__init__(self,op)

self.arg = arg

def __repr__(self):

return "(%s %s)" % (self.op, repr(self.arg))

class DefinedExpr(SingleArgExpr):

def __init__(self,op,macroname):

SingleArgExpr.__init__(self.opDefined,macroname)

def __str__(self):

return "defined(%s)" % self.arg

class UnaryExpr(SingleArgExpr):

def __init__(self,op,arg,opstr=None):

SingleArgExpr.__init__(self,op,arg)

if not opstr:

opstr = op

self.opstr = opstr

def __str__(self):

arg_s = str(self.arg)

arg_prec = self.arg.precedence()

self_prec = self.precedence()

if arg_prec < self_prec:

return "%s(%s)" % (self.opstr,arg_s)

else:

return "%s%s" % (self.opstr, arg_s)

class TwoArgExpr(Expr):

def __init__(self,op,arg1,arg2):

Expr.__init__(self,op)

self.arg1 = arg1

self.arg2 = arg2

def __repr__(self):

return "(%s %s %s)" % (self.op, repr(self.arg1), repr(self.arg2))

class BinaryExpr(TwoArgExpr):

def __init__(self,op,arg1,arg2,opstr=None):

TwoArgExpr.__init__(self,op,arg1,arg2)

if not opstr:

opstr = op

self.opstr = opstr

def __str__(self):

arg1_s = str(self.arg1)

arg2_s = str(self.arg2)

arg1_prec = self.arg1.precedence()

arg2_prec = self.arg2.precedence()

self_prec = self.precedence()

result = ""

if arg1_prec < self_prec:

result += "(%s)" % arg1_s

else:

result += arg1_s

result += " %s " % self.opstr

if arg2_prec < self_prec:

result += "(%s)" % arg2_s

else:

result += arg2_s

return result

#####################################################################################

#####################################################################################

##### #####

##### C P P E X P R E S S I O N P A R S E R #####

##### #####

#####################################################################################

#####################################################################################

class ExprParser:

"""a class used to convert a list of tokens into a cpp Expr object"""

re_octal = re.compile(r"\s*\(0[0-7]+\).*")

re_decimal = re.compile(r"\s*\(\d+[ulUL]*\).*")

re_hexadecimal = re.compile(r"\s*\(0[xX][0-9a-fA-F]*\).*")

def __init__(self,tokens):

self.tok = tokens

self.n = len(self.tok)

self.i = 0

def mark(self):

return self.i

def release(self,pos):

self.i = pos

def peekId(self):

if self.i < self.n:

return self.tok[self.i].id

return None

def peek(self):

if self.i < self.n:

return self.tok[self.i]

return None

def skip(self):

if self.i < self.n:

self.i += 1

def skipOptional(self,id):

if self.i < self.n and self.tok[self.i].id == id:

self.i += 1

def skipSpaces(self):

i = self.i

n = self.n

tok = self.tok

while i < n and (tok[i] == tokSPACE or tok[i] == tokLN):

i += 1

self.i = i

# all the isXXX functions returns a (expr,nextpos) pair if a match is found

# or None if not

def is_integer(self):

id = self.tok[self.i].id

c = id[0]

if c < '0' or c > '9':

return None

m = ExprParser.re_octal.match(id)

if m:

return (IntExpr(id), m.end(1))

m = ExprParser.re_decimal.match(id)

if m:

return (IntExpr(id), m.end(1))

m = ExprParser.re_hexadecimal(id)

if m:

return (IntExpr(id), m.end(1))

return None

def is_defined(self):

id = self.tok[self.i].id

if id != "defined":

return None

pos = self.mark()

use_paren = 0

if self.peekId() == tokLPAREN:

self.skip()

use_paren = 1

if self.peekId() != tokIDENT:

self.throw( BadExpectedToken, "identifier expected")

macroname = self.peek().value

self.skip()

if use_paren:

self.skipSpaces()

if self.peekId() != tokRPAREN:

self.throw( BadExpectedToken, "missing right-paren after 'defined' directive")

self.skip()

i = self.i

return (DefinedExpr(macroname),i+1)

def is_call_or_ident(self):

pass

def parse(self, i):

return None

#####################################################################################

#####################################################################################

##### #####

##### C P P E X P R E S S I O N S #####

##### #####

#####################################################################################

#####################################################################################

class CppInvalidExpression(Exception):

"""an exception raised when an invalid/unsupported cpp expression is detected"""

pass

class CppExpr:

"""a class that models the condition of #if directives into

an expression tree. each node in the tree is of the form (op,arg) or (op,arg1,arg2)

where "op" is a string describing the operation"""

unaries = [ "!", "~" ]

binaries = [ "+", "-", "<", "<=", ">=", ">", "&&", "||", "*", "/", "%", "&", "|", "^", "<<", ">>", "==", "!=" ]

precedences = { "||": 1,

"&&": 2,

"|": 3,

"^": 4,

"&": 5,

"==":6, "!=":6,

"<":7, "<=":7, ">":7, ">=":7,

"<<":8, ">>":8,

"+":9, "-":9,

"*":10, "/":10, "%":10,

"!":11, "~":12

}

def __init__(self, tokens):

"""initialize a CppExpr. 'tokens' must be a CppToken list"""

self.tok = tokens

self.n = len(tokens)

if debugCppExpr:

print "CppExpr: trying to parse %s" % repr(tokens)

expr = self.is_expr(0)

if debugCppExpr:

print "CppExpr: got " + repr(expr)

self.expr = expr[0]

re_cpp_constant = re.compile(r"((\d|\w|_)+)")

def throw(self,exception,i,msg):

if i < self.n:

tok = self.tok[i]

print "%d:%d: %s" % (tok.lineno,tok.colno,msg)

else:

print "EOF: %s" % msg

raise exception

def skip_spaces(self,i):

"""skip spaces in input token list"""

while i < self.n:

t = self.tok[i]