# ***** BEGIN LICENSE BLOCK *****

# Version: MPL 1.1/GPL 2.0/LGPL 2.1

#

# The contents of this file are subject to the Mozilla Public License

# Version 1.1 (the "License"); you may not use this file except in

# compliance with the License. You may obtain a copy of the License at

# http://www.mozilla.org/MPL/

#

# Software distributed under the License is distributed on an "AS IS"

# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the

# License for the specific language governing rights and limitations

# under the License.

#

# The Original Code is Komodo code.

#

# The Initial Developer of the Original Code is ActiveState Software Inc.

# Portions created by ActiveState Software Inc are Copyright (C) 2000-2007

# ActiveState Software Inc. All Rights Reserved.

#

# Contributor(s):

# ActiveState Software Inc

#

# Alternatively, the contents of this file may be used under the terms of

# either the GNU General Public License Version 2 or later (the "GPL"), or

# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),

# in which case the provisions of the GPL or the LGPL are applicable instead

# of those above. If you wish to allow use of your version of this file only

# under the terms of either the GPL or the LGPL, and not to allow others to

# use your version of this file under the terms of the MPL, indicate your

# decision by deleting the provisions above and replace them with the notice

# and other provisions required by the GPL or the LGPL. If you do not delete

# the provisions above, a recipient may use your version of this file under

# the terms of any one of the MPL, the GPL or the LGPL.

#

# ***** END LICENSE BLOCK *****

#

# Contributors:

# Eric Promislow (EricP@ActiveState.com)

"""Ruby parsing support for codeintel/rubycile.py"""

import string

import sys

import re

import textwrap

from SilverCity import ScintillaConstants

from codeintel2 import ruby_lexer

from codeintel2 import shared_lexer

from codeintel2 import shared_parser

from codeintel2 import util

from codeintel2.parser_data import Name_LineNum, VarInfo, Node, ClassNode, \

FileNode, ArgNode, MethodNode, ModuleNode, VariableNode, BlockNode, \

update_collection

from codeintel2.parser_data import VAR_KIND_UNKNOWN, VAR_KIND_GLOBAL, \

VAR_KIND_CLASS, VAR_KIND_CLASSVAR, VAR_KIND_INSTANCE, VAR_KIND_LOCAL, \

VAR_KIND_ALIAS

import logging

log = logging.getLogger("ruby_parser")

# log.setLevel(logging.DEBUG)

# Parse Ruby code

_leading_hash_re = re.compile(r'^\s*\#+\s*')

class RubyCommonClassifier:

"""Mixin class containing classifier callbacks"""

def is_symbol_cb(self, tok):

return tok['text'][0] == ":"

# Used for stripping the quotes off a string

_quote_patterns = {ScintillaConstants.SCE_RB_STRING: re.compile(

'^\"(.*)\"$'),

ScintillaConstants.SCE_RB_CHARACTER: re.compile('^\'(.*)\'$'),

ScintillaConstants.SCE_RB_STRING_QQ: re.compile('^%Q.(.*).$'),

ScintillaConstants.SCE_RB_STRING_Q: re.compile('^%q.(.*).$'),

ScintillaConstants.SCE_RB_DEFAULT: re.compile('^.(.*).$'),

}

def quote_patterns_cb(self, tok):

# Caller wants an array.

return [self.quote_patterns_cb_aux(tok)]

def quote_patterns_cb_aux(self, tok):

tval = tok['text']

if tval[0] == '"':

return self._quote_patterns[ScintillaConstants.SCE_RB_STRING]

elif tval[0] == '\'':

return self._quote_patterns[ScintillaConstants.SCE_RB_CHARACTER]

elif tval.startswith("%Q"):

return self._quote_patterns[ScintillaConstants.SCE_RB_STRING_QQ]

elif tval.startswith("%q"):

return self._quote_patterns[ScintillaConstants.SCE_RB_STRING_Q]

else:

return self._quote_patterns[ScintillaConstants.SCE_RB_DEFAULT] # Fallback

class UDLClassifier(RubyCommonClassifier, shared_parser.UDLClassifier):

pass

class RubyClassifier(RubyCommonClassifier, shared_parser.CommonClassifier):

"""Mixin class containing classifier callbacks"""

def __init__(self):

self.narrowStyles = {

ScintillaConstants.SCE_RB_GLOBAL: VAR_KIND_GLOBAL,

ScintillaConstants.SCE_RB_INSTANCE_VAR: VAR_KIND_INSTANCE,

ScintillaConstants.SCE_RB_CLASS_VAR: VAR_KIND_CLASSVAR}

def get_builtin_type(self, tok, callback):

if self.is_number(tok):

numval = tok['text']

if numval.find(".") >= 0:

return "Float"

else:

return "Fixnum"

elif self.is_string(tok):

return "String"

elif tok['style'] == ScintillaConstants.SCE_RB_STRING_QR:

return "Regexp"

elif tok['style'] == ScintillaConstants.SCE_RB_STRING_QW:

return "Array"

return None

def is_any_operator(self, tok):

return tok['style'] == ScintillaConstants.SCE_RB_OPERATOR

def is_comment(self, tok):

return tok['style'] in (ScintillaConstants.SCE_RB_COMMENTLINE,

ScintillaConstants.SCE_RB_POD)

def is_comment_structured(self, tok, callback):

return tok['style'] == ScintillaConstants.SCE_RB_POD

def is_identifier(self, tok, allow_keywords=False):

return (tok['style'] == ScintillaConstants.SCE_RB_IDENTIFIER or

(allow_keywords and

tok['style'] in [ScintillaConstants.SCE_RB_WORD,

ScintillaConstants.SCE_RB_WORD_DEMOTED]))

def is_interpolating_string(self, tok, callback):

return tok['style'] in [ScintillaConstants.SCE_RB_STRING,

ScintillaConstants.SCE_RB_REGEX,

ScintillaConstants.SCE_RB_HERE_QQ,

ScintillaConstants.SCE_RB_STRING_QQ,

ScintillaConstants.SCE_RB_STRING_QR,

ScintillaConstants.SCE_RB_STRING_QX

]

def is_keyword(self, tok, target):

return tok['style'] == ScintillaConstants.SCE_RB_WORD and tok['text'] == target

def is_number(self, tok):

return tok['style'] == ScintillaConstants.SCE_RB_NUMBER

def is_operator(self, tok, target):

return tok['style'] == ScintillaConstants.SCE_RB_OPERATOR and tok['text'] == target

def is_string(self, tok):

return tok['style'] in [ScintillaConstants.SCE_RB_STRING,

ScintillaConstants.SCE_RB_CHARACTER,

ScintillaConstants.SCE_RB_HERE_Q,

ScintillaConstants.SCE_RB_HERE_QQ,

ScintillaConstants.SCE_RB_STRING_Q,

ScintillaConstants.SCE_RB_STRING_QQ,

ScintillaConstants.SCE_RB_STRING_QX

]

def is_symbol(self, tok, callback=None):

return tok['style'] == ScintillaConstants.SCE_RB_SYMBOL

def tokenStyleToContainerStyle(self, tok, callback):

return self.narrowStyles.get(tok['style'], VAR_KIND_UNKNOWN)

# Accessors for where we'd rather work with a style than call a predicate

# fn

@property

def style_identifier(self):

return ScintillaConstants.SCE_RB_IDENTIFIER

@property

def style_operator(self):

return ScintillaConstants.SCE_RB_OPERATOR

@property

def style_word(self):

return ScintillaConstants.SCE_RB_WORD

lang_specific_classes = {"Ruby": RubyClassifier,

"RHTML": UDLClassifier}

def remove_hashes(lines):

return [_leading_hash_re.sub("", s) for s in lines]

class RailsMigrationData:

def __init__(self):

self.tableHookName = None

self.columns = [] # Array of {name, type} pairs

class RailsMigrationBlock:

def __init__(self):

self.table_name = None

# non-neg means we're in the pertinent scope

self.class_indentLevel = -1

self.upFunc_indentLevel = -1

self.data = [] # Array of RailsMigrationData

_inflector = None

def get_inflector():

global _inflector

if _inflector is None:

import inflector.Inflector

_inflector = inflector.Inflector.Inflector()

return _inflector

class Parser:

# New names could be added to Rails post Rails 2.0,

# for database-specific types like 'blob'

_rails_migration_types = ('binary', 'boolean', 'date',

'datetime', 'decimal', 'float',

'string', 'integer',

)

def __init__(self, tokenizer, lang):

self.tokenizer = tokenizer

self.block_stack = []

self.tree = FileNode()

self.curr_node = self.tree

self.bracket_matchers = {"[": "]", "{": "}", "(": ")"}

self.classifier = lang_specific_classes[lang]()

self.containers = {VAR_KIND_GLOBAL: [self.tree.global_vars],

VAR_KIND_CLASS: [], # classes

VAR_KIND_CLASSVAR: [],

VAR_KIND_INSTANCE: [],

VAR_KIND_LOCAL: [self.tree.local_vars], # locals

VAR_KIND_ALIAS: [self.tree.aliases],

}

self.rails_migration_block = RailsMigrationBlock()

def class_has_method(self, curr_node, the_text):

try:

class_node = self.containers[VAR_KIND_CLASS][-1]

for c in class_node.children:

if isinstance(c, MethodNode) and c.name == the_text:

return True

except:

pass

return False

def curr_block_start_indentation(self):

try:

ind = self.block_stack[-1].indentation

except:

ind = 0

return ind

# This routine compares the current line indentation

# with the one at the start of the block

#

# Neg value: we've moved to far back

# 0: same ind

# Pos value: curr indent'n is greater than where we started

def compare_curr_ind(self):

start_ind = self.curr_block_start_indentation()

curr_indent = self.tokenizer.get_curr_indentation()

return curr_indent - start_ind

def dump(self):

self.tree.dump()

def rails_migration_class_tree(self):

rails_migration_block = self.rails_migration_block

inflector = get_inflector()

nodes = []

for info in self.rails_migration_block.data:

if info.tableHookName is None or not info.columns:

return None

className = inflector.camelize(

inflector.singularize(info.tableHookName))

classNode = ClassNode(className, info.startLine, False)

classNode.set_line_end_num(info.endLine)

for attrib_info in info.columns:

methodNode = MethodNode(attrib_info[0], attrib_info[2])

methodNode.set_line_end_num(attrib_info[2])

classNode.append_node(methodNode)

nodes.append(classNode)

return nodes

def parse(self):

self.parse_aux(self.tree)

return self.tree

def get_parsing_objects(self, kwd):

return {

"module": [ModuleNode, self.parse_aux],

"class": [ClassNode, self.parse_aux],

"def": [MethodNode, self.parse_method]

}.get(kwd, [None, None])

def parse_open_parens(self):

paren_count = 0

while True:

tok = self.tokenizer.get_next_token()

if self.classifier.is_operator(tok, "("):

paren_count += 1

else:

return (tok, paren_count)

def parse_attr_stmt_capture_info(self, collection, attr_tok, tok, curr_node):

base_name = tok['text'][1:]

update_collection(collection,

'@' + base_name, tok['start_line'])

if isinstance(curr_node, ClassNode):

if attr_tok != 'attr_writer':

new_node = MethodNode(base_name, tok['start_line'])

new_node.set_line_end_num(tok['start_line'])

curr_node.append_node(new_node)

if attr_tok in ['attr_writer', 'attr_accessor']:

new_node = MethodNode(base_name + '=', tok['start_line'])

new_node.set_line_end_num(tok['start_line'])

new_node.add_arg(base_name, "")

curr_node.append_node(new_node)

def parse_attr_stmts(self, attr_tok, curr_node):

"""

attr_tok is one of ['attr', 'attr_reader', 'attr_writer', 'attr_accessor']

if curr_node is a ClassNode then set up a bunch of methods as well

"""

try:

collection = self.containers[VAR_KIND_INSTANCE][-1]

if collection is None:

return

except:

return

(tok, paren_count) = self.parse_open_parens()

if self.classifier.is_symbol(tok, self.classifier.is_symbol_cb):

self.parse_attr_stmt_capture_info(

collection, attr_tok, tok, curr_node)

# Allow for more, but special case attr's

while True:

tok = self.tokenizer.get_next_token()

if self.classifier.is_operator(tok, ","):

tok = self.tokenizer.get_next_token()

# Intern unless attr

if (self.classifier.is_symbol(tok, self.classifier.is_symbol_cb) and

attr_tok != 'attr'):

self.parse_attr_stmt_capture_info(

collection, attr_tok, tok, curr_node)

else:

self.tokenizer.put_back(tok)

break

# Might as well consume the closing parens

while paren_count > 0:

tok = self.tokenizer.get_next_token()

if self.classifier.is_operator(tok, ")"):

paren_count -= 1

else:

self.tokenizer.put_back(tok)

break

def parse_assignment(self, collectionA, tok_text, start_line):

if len(collectionA) == 0 or collectionA[-1] is None:

return

tok = self.tokenizer.get_next_token()

# We don't need to Watch out for calls like abc=(1)

# because that's always an assignment to a local.

# self.abc=(1) is different

if self.classifier.is_any_operator(tok):

if tok['text'] == "=":

self._finishVarAssignment(collectionA, tok_text, start_line)

# Look to see if we have an instance of a class

# This is

update_collection(collectionA[-1], tok_text, start_line)

return

elif tok['text'] in ["::", "."]:

# Don't store fully-qualified names, but do consume them

# Keep name for debugging purposes.

rest_of_name = self.get_fully_qualified_name()

return

self.tokenizer.put_back(tok)

def _actual_string_from_string_or_symbol(self, tok):

if self.classifier.is_symbol(tok):

return tok['text'][1:]

else:

assert self.classifier.is_string(tok)

return self.de_quote_string(tok)

def _parse_migration_create_table_block(self):

"""This code is handled only when in the following conditions:

1. The file path matches .../db/migrate/*.rb

2. It's in a class that inherits from ActiveRecord::Migration

3. It's in a function called "self.up"

Database fields are found through two functions:

create_table TABLE-NAME do |handle|

handle.column COLUMN-NAME TYPE

end

To add:

add_column TABLE-NAME COLUMN-NAME TYPE

"""

tok = self.tokenizer.get_next_token()

if not (self.classifier.is_symbol(tok) or self.classifier.is_string(tok)):

return

rails_migration_info = RailsMigrationData()

rails_migration_info.tableHookName = self._actual_string_from_string_or_symbol(

tok)

rails_migration_info.startLine = tok['start_line']

# Assume we don't find the end of the block, so assume the worst

rails_migration_info.endLine = tok['start_line'] + 1

tok = self.tokenizer.get_next_token()

if self.classifier.is_operator(tok, "{") or self.classifier.is_keyword(tok, "do"):

control_tok = (tok['style'], (tok['text'] == "{" and "}") or "end")

else:

return

tok = self.tokenizer.get_next_token()

if not self.classifier.is_operator(tok, "|"):

return

tok = self.tokenizer.get_next_token()

if not self.classifier.is_identifier(tok):

return

table_handle = tok['text']

tok = self.tokenizer.get_next_token()

if not self.classifier.is_operator(tok, "|"):

return

add_entry = False

while True:

tok = self.tokenizer.get_next_token()

if tok['style'] == shared_lexer.EOF_STYLE:

break

elif tok['style'] == control_tok[0] and tok['text'] == control_tok[1]:

rails_migration_info.endLine = tok['end_line']

break

elif self.classifier.is_identifier(tok) and tok['text'] == table_handle:

tok = self.tokenizer.get_next_token()

if self.classifier.is_operator(tok, "."):

tok = self.tokenizer.get_next_token()

if self.classifier.is_identifier(tok):

line_num = tok['start_line']

if tok['text'] == 'column':

tok = self.tokenizer.get_next_token()

if self.classifier.is_symbol(tok) or self.classifier.is_string(tok):

column_name = self._actual_string_from_string_or_symbol(

tok)

tok = self.tokenizer.get_next_token()

if self.classifier.is_operator(tok, ','):

tok = self.tokenizer.get_next_token()

if self.classifier.is_symbol(tok) or self.classifier.is_string(tok):

column_type = self._actual_string_from_string_or_symbol(

tok)

else:

column_type = ""

add_entry = True

elif tok['text'] in self._rails_migration_types:

column_type = tok['text']

tok = self.tokenizer.get_next_token()

log.debug("Migration: add %s:%s",

tok['text'], column_type)

if self.classifier.is_symbol(tok) or self.classifier.is_string(tok):

column_name = self._actual_string_from_string_or_symbol(

tok)

add_entry = True

elif tok['text'] == "timestamps":

# Rails 2.0 migration syntax in these two blocks:

# create_table :objects do |t|

# t.<type> :<name>

column_type = 'datetime'

rails_migration_info.columns.append((

'created_at', column_type, line_num))

rails_migration_info.endLine = line_num + 1

column_name = 'updated_at'

add_entry = True

if add_entry:

rails_migration_info.columns.append(

(column_name, column_type, line_num))

rails_migration_info.endLine = line_num + 1

add_entry = False

continue

# Sanity check any token we have here

if (tok['style'] == ScintillaConstants.SCE_RB_WORD and

tok['text'] in ('class', 'def')):

self.tokenizer.put_back(tok)

return

self.rails_migration_block.data.append(rails_migration_info)

# end _parse_migration_create_table_block

def _parse_migration_add_column_stmt(self):

"""This code is handled only when in the following conditions:

1. The file path matches .../db/migrate/*.rb

2. It's in a class that inherits from ActiveRecord::Migration

3. It's in a function called "self.up"

4. We saw an add_column action.

Syntax:

add_column TABLE-NAME COLUMN-NAME TYPE

"""

tok = self.tokenizer.get_next_token()

if not (self.classifier.is_symbol(tok) or self.classifier.is_string(tok)):

return

table_name = self._actual_string_from_string_or_symbol(tok)

tok = self.tokenizer.get_next_token()

if not self.classifier.is_operator(tok, ","):

return

tok = self.tokenizer.get_next_token()

if not (self.classifier.is_symbol(tok) or self.classifier.is_string(tok)):

return

rails_migration_info = RailsMigrationData()

rails_migration_info.tableHookName = table_name

rails_migration_info.endLine = rails_migration_info.startLine = tok[

'start_line']

column_name = self._actual_string_from_string_or_symbol(tok)

column_type = ""

tok = self.tokenizer.get_next_token()

if self.classifier.is_operator(tok, ','):

tok = self.tokenizer.get_next_token()

if self.classifier.is_symbol(tok) or self.classifier.is_string(tok):

column_type = self._actual_string_from_string_or_symbol(tok)

else:

self.tokenizer.put_back(tok)

else:

self.tokenizer.put_back(tok)

rails_migration_info.columns.append((

column_name, column_type, rails_migration_info.startLine))

self.rails_migration_block.data.append(rails_migration_info)

def _finish_interpolating_string(self, prev_tok):

""" Matches sequences of "#{", ... "}"

(which could nest). Returns a list of all the tokens.

"""

tok1 = self.tokenizer.get_next_token()

if not self.classifier.is_any_operator(tok1) or \

tok1['text'] != "#":

self.tokenizer.put_back(tok1)

return [prev_tok]

tok2 = self.tokenizer.get_next_token()

if not self.classifier.is_any_operator(tok2) or \

tok2['text'] != "{":

self.tokenizer.put_back(tok1)

self.tokenizer.put_back(tok2)

return [prev_tok]

nested_count = 1

tok_list = [prev_tok, tok1, tok2]

# At this point process a contiguous stream of tokens

while True:

bail_out = False

new_tok = self.tokenizer.get_next_token()

if new_tok['style'] == shared_lexer.EOF_STYLE:

bail_out = True

elif self.classifier.is_any_operator(new_tok):

if new_tok['text'] == "#":

tok_list.append(new_tok)

new_tok = self.tokenizer.get_next_token()

if self.classifier.is_any_operator(new_tok) and \

new_tok['text'] == "{":

nested_count += 1

elif nested_count == 0:

bail_out = True

elif new_tok['text'] == "}" and nested_count > 0:

nested_count -= 1

elif nested_count == 0:

bail_out = True

elif self.classifier.is_interpolating_string(new_tok, self._test_interpolate_string):

pass

elif nested_count == 0:

bail_out = True

if bail_out:

self.tokenizer.put_back(new_tok)

return tok_list

else:

tok_list.append(new_tok)

def _at_end_expression(self, tok):

if not self.classifier.is_any_operator(tok):

return True

return tok['text'] in [",", "}", ";"]

def skip_to_op(self, opText, nestedLevel=1):

skipped_toks = []

while 1:

tok = self.tokenizer.get_next_token()

skipped_toks.append(tok)

if tok['style'] == shared_lexer.EOF_STYLE:

return skipped_toks

elif self.classifier.is_any_operator(tok):

if tok['text'] in ["(", "{", "["]:

nestedLevel += 1

elif tok['text'] in ["]", "}", ")"]:

nestedLevel -= 1

if nestedLevel <= 0:

return skipped_toks

# don't care about other ops

# don't care about other token types

def de_quote_string(self, tok):

tval = tok['text']

patterns = self.classifier.get_quote_patterns(

tok, self.classifier.quote_patterns_cb)

for p in patterns:

m = p.match(tval)

if m:

return m.group(1)

return tval

def _finishVarAssignment(self, collectionA, var_name, start_line):

# See doc for finishVarAssignment in src/perlcile/ParseModule.pm

# for comments.

# 1: a = b.c.d.e -- don't bother

# 2.1: a = <number> <no op>

# 2.2: a = <string> <no op>

# 3. a = <classname>.New [( ... )]<no op>

tok_list = self.get_fully_qualified_name_as_list()

if not tok_list:

return

tok = tok_list[0]

if self.classifier.is_identifier(tok):

var_citdl = None

toks_to_return = tok_list

if len(toks_to_return) < 3:

tok2 = self.tokenizer.get_next_token()

# toks_to_return.append(tok2)

if self.classifier.is_operator(tok2, ','):

# XXX : skip to end -- we must be in a parallel assignment

pass

else:

if self._at_end_expression(tok2):

var_citdl = tok['text']

self.tokenizer.put_back(tok2)

elif self.classifier.is_operator(toks_to_return[-2], '.') and \

self.classifier.is_identifier(toks_to_return[-1]) and \

toks_to_return[-1]['text'] == 'new':

tok3 = self.tokenizer.get_next_token()

if self._at_end_expression(tok3):

var_citdl = ''.join([tok[

'text'] for tok in tok_list[:-2]])

self.tokenizer.put_back(tok3)

# toks_to_return.append(tok3)

else:

if self.classifier.is_operator(tok3, "("):

skipped_toks = self.skip_to_op(")")

# toks_to_return += skipped_toks

tok4 = self.tokenizer.get_next_token()

else:

tok4 = tok3

if self._at_end_expression(tok4):

var_citdl = ''.join([tok[

'text'] for tok in tok_list[:-2]])

self.tokenizer.put_back(tok4)

update_collection(collectionA[-1], var_name, start_line, var_citdl)

# Idea: don't bother putting back these tokens

# for t in tok_list:

# self.tokenizer.put_back(t)

return

builtin_type = self.classifier.get_builtin_type(

tok, self._test_for_builtin_type)

if builtin_type:

type1, locn = tok['style'], tok['start_line']

if self.classifier.is_interpolating_string(tok, self._test_interpolate_string):

self._finish_interpolating_string(tok)

tok2 = self.tokenizer.get_next_token()

if self._at_end_expression(tok2):

if self.classifier.is_number(tok):

type1 = "Number"

else:

type1 = "String"

update_collection(collectionA[

-1], var_name, start_line, builtin_type)

toks = [tok2]

else:

update_collection(collectionA[-1], var_name, start_line)

toks = [tok2, tok]

elif tok['style'] == self.classifier.style_identifier:

raise(

"get_fully_qualified_name_as_list failed to process an identifer")

elif self.classifier.is_operator(tok, "["):

toks = self._finish_list(

collectionA, var_name, start_line, tok, "]", "Array")

elif self.classifier.is_operator(tok, "{"):

toks = self._finish_list(

collectionA, var_name, start_line, tok, "}", "Hash")

else:

update_collection(collectionA[-1], var_name, start_line)

toks = [tok]

for t in reversed(toks):

self.tokenizer.put_back(t)

def _finish_list(self, collectionA, var_name, start_line, orig_tok, end_op, class_name):

skipped_toks = [orig_tok] + self.skip_to_op(end_op)

final_tok = self.tokenizer.get_next_token()

skipped_toks.append(final_tok)

class_name_final = self._at_end_expression(

final_tok) and class_name or None

update_collection(collectionA[

-1], var_name, start_line, class_name_final)

return skipped_toks

def _set_basename_method(self, node_class, curr_node, nm_token):

if node_class != MethodNode:

return (nm_token, False)

method_name = nm_token[0]

idx1 = method_name.rfind('::')

if idx1 >= 0:

idx1_len = 2

idx2 = method_name.rfind('.')

if idx2 >= 0:

idx2_len = 1

if idx1 < 0:

if idx2 < 0:

return (nm_token, False)

idx = idx2

idx_len = 1

elif idx2 < idx1:

# Includes no idx2

idx = idx1

idx_len = 2

else:

idx = idx2

idx_len = 1

first_part = method_name[0:idx]

if first_part != getattr(curr_node, 'name', '') and first_part != 'self':

# We're doing something like defining inside a mix-in

return (nm_token, False)

basename = method_name[idx + idx_len:]

if len(nm_token) != 2:

raise("Expectations dashed!")

return ((basename, nm_token[1]), True)

def _test_interpolate_string(self, tok, generic_tok_type):

if generic_tok_type == shared_parser.GENERIC_TYPE_REGEX:

return True

elif generic_tok_type != shared_parser.GENERIC_TYPE_STRING:

return False

tval = tok['text']

c1 = tval[0]

if c1 == "'":

return False

elif c1 == '"':

return True

elif c1 == '%':

if len(c1) == 1:

return False # ???

c2 = tval[1]

if c2 in "WwQrx":

return True

return False

# Callback used by get_builtin_type

def _test_for_builtin_type(self, tok, generic_tok_type):

if generic_tok_type == shared_parser.GENERIC_TYPE_NUMBER:

numval = tok['text']

if numval.find(".") >= 0:

return "Float"

else:

return "Fixnum"

elif generic_tok_type == shared_parser.GENERIC_TYPE_STRING:

tval = tok['text']

if len(tval) > 1 and tval[0] == "%":

if tval[1] in 'wW':

return "Array"

elif tval[1] == 'r':

return "Regexp"

return "String"

elif generic_tok_type == shared_parser.GENERIC_TYPE_REGEX:

return "Regexp"

else:

return None

# Callback used by tokenStyleToContainerStyle

def _test_token_style(self, tok, is_variable_token):

if not is_variable_token:

return VAR_KIND_UNKNOWN

tval = tok['text']

if tval[0] == '$':

return VAR_KIND_GLOBAL

elif tval[0] == '@':

if len(tval) > 1 and tval[1] == '@':

return VAR_KIND_CLASSVAR

else:

return VAR_KIND_INSTANCE

return VAR_KIND_UNKNOWN

def _try_loading_relative_library(self, tok, curr_node):

"""Look for instances of

require File.dirname(__FILE__) + <string> and map to

@<string>"""

seq_start_line = tok['start_line']

if not self.classifier.is_identifier(tok) or tok['text'] != 'File':

return

tok = self.tokenizer.get_next_token()

if not self.classifier.is_operator(tok, '.'):

return

tok = self.tokenizer.get_next_token()

if not self.classifier.is_identifier(tok) or tok['text'] != 'dirname':

return

tok = self.tokenizer.get_next_token()

if not self.classifier.is_operator(tok, '('):

return

tok = self.tokenizer.get_next_token()

if not self.classifier.is_identifier(tok, True) or tok['text'] != '__FILE__':

return

tok = self.tokenizer.get_next_token()

if not self.classifier.is_operator(tok, ')'):

return

tok = self.tokenizer.get_next_token()

if not self.classifier.is_operator(tok, '+'):

return

tok = self.tokenizer.get_next_token()

if self.classifier.is_string(tok):

curr_node.imports.append(Name_LineNum

(self._create_relative_libname(tok),

seq_start_line))

def _create_relative_libname(self, tok):

# For now just return the path. If we need to decorate it

# to indicate it's a relative path, we'll do that later.

return self.de_quote_string(tok)[1:]

def _extract_alias_name(self, tok):

if self.classifier.is_identifier(tok):

return tok['text']

elif self.classifier.is_symbol(tok, self.classifier.is_symbol_cb):

return tok['text'][1:]

else:

return None

def parse_aux(self, curr_node):

at_start = True

while 1:

tok = self.tokenizer.get_next_token()

if tok['style'] == shared_lexer.EOF_STYLE:

break

# style, text, start_column, start_line, end_column, end_line = tok

style, text = tok['style'], tok['text']

if style == self.classifier.style_word:

if text in ["module", "class", "def"]:

# Check to see if we missed some nodes,

# using heuristic that the end statements line up

# with the start of each block, for definitions

if len(self.block_stack) > 0 and self.compare_curr_ind() <= 0:

# We missed one or more end's, so put the token

# back and try in an outer instance

self.tokenizer.put_back(tok)

return

curr_indent = self.tokenizer.get_curr_indentation()

node_class, node_parser = self.get_parsing_objects(text)

if node_class is None:

sys.stderr.write(

"Couldn't get parsing objects for type %s\n" % text)

break

elif node_class is MethodNode:

self.tokenizer.start_sig()

# Get the comments before further parsing.

comment_lines = remove_hashes(

self.tokenizer.curr_comment())

nm_token = self.get_fully_qualified_name()

if not nm_token[0]:

return

elif nm_token[0] == "<<" and node_class == ClassNode:

# Singleton classes

nm_token = self.get_fully_qualified_name()

if not nm_token[0]:

return

if self.rails_migration_block.class_indentLevel >= 0 and nm_token[0] == "self.up":

self.rails_migration_block.upFunc_indentLevel = curr_indent

rails_migration_clear_upfunc = True

else:

rails_migration_clear_upfunc = False

(nm_token, is_class_method) = self._set_basename_method(

node_class, curr_node, nm_token)

new_node = node_class(nm_token[0], tok[

'start_line'], nm_token[0] == "initialize")

if is_class_method:

new_node.is_classmethod = True

elif node_class == MethodNode and (isinstance(curr_node, ClassNode) or isinstance(curr_node, ModuleNode)):

# Add info on the 'self' variable

update_collection(new_node.local_vars, 'self', tok[

'start_line'], curr_node.name)

new_node.doc_lines = comment_lines

new_node.indentation = curr_indent

self.block_stack.append(new_node)

curr_node.append_node(new_node)

# Push new containers on the symbol table

self.containers[VAR_KIND_LOCAL].append(new_node.local_vars)

if node_class in [ClassNode, ModuleNode]:

self.containers[VAR_KIND_CLASS].append(new_node)

self.containers[VAR_KIND_CLASSVAR].append(

new_node.class_vars)

self.containers[VAR_KIND_INSTANCE].append(

new_node.instance_vars)

self.containers[VAR_KIND_ALIAS].append(

new_node.aliases)

# Watch out for class inheritence

rails_migration_clear_classref = False

if node_class == ClassNode:

self.parse_classref(new_node)

if new_node.has_classref('ActiveRecord::Migration'):

self.rails_migration_block.class_indentLevel = curr_indent

rails_migration_clear_classref = True

node_parser(new_node) # Has self bound to it

if rails_migration_clear_upfunc:

self.rails_migration_block.upFunc_indentLevel = -1

elif rails_migration_clear_classref:

self.rails_migration_block.class_indentLevel = -1

self.block_stack.pop()

self.containers[VAR_KIND_LOCAL].pop()

if node_class in [ClassNode, ModuleNode]:

self.containers[VAR_KIND_CLASSVAR].pop()

self.containers[VAR_KIND_INSTANCE].pop()

self.containers[VAR_KIND_CLASS].pop()

self.containers[VAR_KIND_ALIAS].pop()

# Clear any comment that's hanging around

self.tokenizer.clear_comments()

elif text == "alias":

new_tok = self.tokenizer.get_next_token()

new_name = self._extract_alias_name(new_tok)

existing_tok = self.tokenizer.get_next_token()

existing_name = self._extract_alias_name(existing_tok)

if new_name and existing_name:

update_collection(self.containers[VAR_KIND_ALIAS][

-1], new_name, new_tok['start_line'], existing_name)

# set a variable in curr_node

elif text == "end":

if len(self.block_stack) > 0:

end_position = self.compare_curr_ind()

if end_position < 0:

# We've gone too far, so put it back and try later

curr_node.set_line_end_num(tok['start_line'])

self.tokenizer.put_back(tok)

return

elif end_position == 0:

curr_node.set_line_end_num(tok['start_line'])

# the caller will pop the stack

return

elif style == self.classifier.style_identifier:

if text == "require" or text == "load":

tok = self.tokenizer.get_next_token()

if (self.tokenizer.is_string_token(tok)):

tval = self.de_quote_string(tok)

if tval.endswith(".rb"):

tval = tval[:-3]

curr_node.imports.append(

Name_LineNum(tval, tok['start_line']))

else:

self._try_loading_relative_library(tok, curr_node)

elif text == "include":

nm_token = self.get_fully_qualified_name()

if nm_token[0]:

curr_node.includes.append(Name_LineNum(

nm_token[0], nm_token[1]))

#@@@@ elif text == "base":

# semi-hardwired rails thing

#@@@@ self.parse_class_extension()

# if next() == (OP, ".") \

# and next() == (ID, "extend") \

# and next() == (OP, "(") \

elif at_start or tok['start_column'] == self.tokenizer.get_curr_indentation():

# Look at these things only at start of line

if text in ['attr', 'attr_reader', 'attr_writer', 'attr_accessor']:

self.parse_attr_stmts(text, curr_node)

elif text == 'create_table' and self.rails_migration_block.upFunc_indentLevel >= 0:

self._parse_migration_create_table_block()

elif text == 'add_column' and self.rails_migration_block.upFunc_indentLevel >= 0:

self._parse_migration_add_column_stmt()

elif not self.class_has_method(curr_node, tok['text'] + "="):

# Make sure it isn't an assignment function

self.parse_assignment(self.containers[

VAR_KIND_LOCAL], text, tok['start_line'])