# ***** BEGIN LICENSE BLOCK *****

# Version: MPL 1.1/GPL 2.0/LGPL 2.1

#

# The contents of this file are subject to the Mozilla Public License

# Version 1.1 (the "License"); you may not use this file except in

# compliance with the License. You may obtain a copy of the License at

# http://www.mozilla.org/MPL/

#

# Software distributed under the License is distributed on an "AS IS"

# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the

# License for the specific language governing rights and limitations

# under the License.

#

# The Original Code is Komodo code.

#

# The Initial Developer of the Original Code is ActiveState Software Inc.

# Portions created by ActiveState Software Inc are Copyright (C) 2000-2007

# ActiveState Software Inc. All Rights Reserved.

#

# Contributor(s):

# ActiveState Software Inc

#

# Alternatively, the contents of this file may be used under the terms of

# either the GNU General Public License Version 2 or later (the "GPL"), or

# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),

# in which case the provisions of the GPL or the LGPL are applicable instead

# of those above. If you wish to allow use of your version of this file only

# under the terms of either the GPL or the LGPL, and not to allow others to

# use your version of this file under the terms of the MPL, indicate your

# decision by deleting the provisions above and replace them with the notice

# and other provisions required by the GPL or the LGPL. If you do not delete

# the provisions above, a recipient may use your version of this file under

# the terms of any one of the MPL, the GPL or the LGPL.

#

# ***** END LICENSE BLOCK *****

#

# Contributors:

# Eric Promislow (EricP@ActiveState.com)

"""Perl parsing support for codeintel/perlcile.py"""

import copy

import os.path

import string

import sys

import re

import textwrap

import time

import pickle

import logging

from ciElementTree import Element, SubElement, tostring

from SilverCity import ScintillaConstants

from SilverCity.ScintillaConstants import (

SCE_PL_DEFAULT, SCE_PL_ERROR, SCE_PL_COMMENTLINE, SCE_PL_POD,

SCE_PL_NUMBER, SCE_PL_WORD, SCE_PL_STRING, SCE_PL_CHARACTER,

SCE_PL_PUNCTUATION, SCE_PL_PREPROCESSOR, SCE_PL_OPERATOR,

SCE_PL_IDENTIFIER, SCE_PL_SCALAR, SCE_PL_ARRAY, SCE_PL_HASH,

SCE_PL_SYMBOLTABLE, SCE_PL_VARIABLE_INDEXER, SCE_PL_REGEX,

SCE_PL_REGSUBST, SCE_PL_LONGQUOTE, SCE_PL_BACKTICKS, SCE_PL_DATASECTION,

SCE_PL_HERE_DELIM, SCE_PL_HERE_Q, SCE_PL_HERE_QQ, SCE_PL_HERE_QX,

SCE_PL_STRING_Q, SCE_PL_STRING_QQ, SCE_PL_STRING_QX, SCE_PL_STRING_QR,

SCE_PL_STRING_QW, SCE_PL_POD_VERB, SCE_PL_SUB, SCE_PL_SUB_ARGS,

SCE_PL_UNKNOWN_FIELD, SCE_PL_STDIN, SCE_PL_STDOUT, SCE_PL_STDERR,

SCE_PL_FORMAT, SCE_PL_UPPER_BOUND)

from codeintel2.common import CILEError

from codeintel2 import perl_lexer

from codeintel2 import shared_lexer

from codeintel2 import shared_parser

SCE_PL_UNUSED = shared_lexer.EOF_STYLE

log = logging.getLogger("perlcile")

# log.setLevel(logging.DEBUG)

#---- memoize from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/496879

TIMING = False # set to true to capture timing data from regexen

REGEXEN = {} # unused if TIMING is not True

def memoize(function, limit=None):

if isinstance(function, int):

def memoize_wrapper(f):

return memoize(f, function)

return memoize_wrapper

dict = {}

list = []

def memoize_wrapper(*args, **kwargs):

key = pickle.dumps((args, kwargs))

try:

list.append(list.pop(list.index(key)))

except ValueError:

dict[key] = function(*args, **kwargs)

list.append(key)

if limit is not None and len(list) > limit:

del dict[list.pop(0)]

return dict[key]

memoize_wrapper._memoize_dict = dict

memoize_wrapper._memoize_list = list

memoize_wrapper._memoize_limit = limit

memoize_wrapper._memoize_origfunc = function

memoize_wrapper.__name__ = function.__name__

return memoize_wrapper

class TimingRe:

"A wrapper around compiled regexen that keeps track of timing data"

def __init__(self, re, orig_re):

self._re = re

self._orig_re = orig_re

def sub(self, *args):

return self._timing_operation('sub', *args)

def match(self, *args):

return self._timing_operation('match', *args)

def search(self, *args):

return self._timing_operation('search', *args)

def split(self, *args):

return self._timing_operation('split', *args)

def findall(self, *args):

return self._timing_operation('findall', *args)

def _timing_operation(self, methodname, *args):

start = time.time()

retval = getattr(self._re, methodname)(*args)

end = time.time()

delta = end-start

# if delta > 0.01:

# print delta,'\t', self._orig_re, str(args)[:80]

if self._orig_re not in REGEXEN:

REGEXEN[self._orig_re] = 0.0

REGEXEN[self._orig_re] += delta

return retval

@memoize

def re_compile(regex, *args):

"""A version of re.compile which memoizes and optionally keeps track of timing

data"""

if TIMING:

return TimingRe(re.compile(regex, *args), regex)

else:

return re.compile(regex, *args)

def re_sub(*args):

"""a version of re.sub which deals with TimingRe objects and prints out

details of slow regexen"""

if TIMING:

start = time.time()

if isinstance(args[0], TimingRe):

retval = args[0].sub(*args[1:])

else:

retval = re.sub(*args)

end = time.time()

delta = end-start

if delta > 0.01: # adjust as needed.

print(delta, '\t', args)

else:

retval = re.sub(*args)

return retval

class PerlCommonClassifier:

"""Mixin class containing classifier callbacks"""

def is_array_cb(self, tok):

tval = tok['text']

return len(tval) >= 2 and tval[0] == '@' and tval[1] != '$'

# @$name is more like an expression -- don't return it

def is_scalar_cb(self, tok):

tval = tok['text']

return len(tval) > 1 and tval[0] == '$' and (tval[1].isalnum or tval[1] == "_")

def is_pod_cb(self, tok):

return tok['text'][0] == '=' and tok['text'][1].isalnum and tok['text'].find("\n=cut", 5) > 0

def is_string_qw_cb(self, tok):

return re_compile(r'^qw\s*[^\w\d_]').match(tok['text'])

# Used for stripping the quotes off a string

_quote_patterns = {SCE_PL_STRING: re.compile('^[\'\"](.*)[\'\"]$'),

SCE_PL_CHARACTER: re.compile('^\'(.*)\'$'),

SCE_PL_STRING_Q: re.compile(r'^q\s*.(.*).$'),

SCE_PL_STRING_QQ: re.compile(r'^q\w\s*.(.*).$'),

SCE_PL_DEFAULT: re.compile('^.(.*).$'), # fallback

}

def quote_patterns_cb(self, tok):

# Caller wants an array.

return [self.quote_patterns_cb_aux(tok)]

def quote_patterns_cb_aux(self, tok):

tval = tok['text']

if tval[0] == '"':

return self._quote_patterns[SCE_PL_STRING]

elif tval[0] == '\'':

return self._quote_patterns[SCE_PL_CHARACTER]

elif tval.startswith("Q"):

return self._quote_patterns[SCE_PL_STRING_QQ]

elif tval.startswith("q"):

return self._quote_patterns[SCE_PL_STRING_Q]

else:

return self._quote_patterns[SCE_PL_DEFAULT] # Fallback

class UDLClassifier(PerlCommonClassifier, shared_parser.UDLClassifier):

pass

class PerlClassifier(PerlCommonClassifier, shared_parser.CommonClassifier):

def get_builtin_type(self, tok, callback):

raise CILEError("Unexpected call to perl_parser.get_builtin_type")

def is_any_operator(self, tok):

return tok['style'] == ScintillaConstants.SCE_PL_OPERATOR

def is_comment(self, tok):

return tok['style'] in (ScintillaConstants.SCE_PL_COMMENT,

ScintillaConstants.SCE_PL_POD)

def is_comment_structured(self, tok, callback):

return tok['style'] == ScintillaConstants.SCE_PL_POD

def is_identifier(self, tok, allow_keywords=False):

return (tok['style'] == ScintillaConstants.SCE_PL_IDENTIFIER or

(allow_keywords and

tok['style'] == ScintillaConstants.SCE_PL_WORD))

def is_index_op(self, tok, pattern=None):

if not (tok['style'] in (SCE_PL_OPERATOR, SCE_PL_VARIABLE_INDEXER)):

return False

elif not pattern:

return True

return len(tok['text']) > 0 and pattern.search(tok['text'])

def is_interpolating_string(self, tok, callback):

return tok['style'] in [ScintillaConstants.SCE_PL_STRING,

ScintillaConstants.SCE_PL_REGEX,

ScintillaConstants.SCE_PL_HERE_QQ,

ScintillaConstants.SCE_PL_STRING_QQ,

ScintillaConstants.SCE_PL_STRING_QR,

ScintillaConstants.SCE_PL_STRING_QX

]

def is_keyword(self, tok, target):

return tok['style'] == ScintillaConstants.SCE_PL_WORD and tok['text'] == target

def is_number(self, tok):

return tok['style'] == ScintillaConstants.SCE_PL_NUMBER

def is_operator(self, tok, target):

return tok['style'] == ScintillaConstants.SCE_PL_OPERATOR and tok['text'] == target

def is_string(self, tok):

return tok['style'] in [ScintillaConstants.SCE_PL_STRING,

ScintillaConstants.SCE_PL_CHARACTER,

ScintillaConstants.SCE_PL_HERE_Q,

ScintillaConstants.SCE_PL_HERE_QQ,

ScintillaConstants.SCE_PL_STRING_Q,

ScintillaConstants.SCE_PL_STRING_QQ,

ScintillaConstants.SCE_PL_STRING_QX,

]

def is_string_qw(self, tok, callback):

return tok['style'] == ScintillaConstants.SCE_PL_STRING_QW

def is_symbol(self, tok):

return False

def is_variable(self, tok):

return SCE_PL_SCALAR <= tok['style'] <= SCE_PL_SYMBOLTABLE

# Types of variables

def is_variable_array(self, tok, callback=None):

return tok['style'] == ScintillaConstants.SCE_PL_ARRAY and \

len(tok['text']) > 1 and tok['text'][1] != '$'

def is_variable_scalar(self, tok, callback=None):

return tok['style'] == ScintillaConstants.SCE_PL_SCALAR and \

len(tok['text']) > 1 and tok['text'][1] != '$'

# Accessors for where we'd rather work with a style than call a predicate

# fn

@property

def style_identifier(self):

return ScintillaConstants.SCE_PL_IDENTIFIER

@property

def style_word(self):

return ScintillaConstants.SCE_PL_WORD

def _get_classifier(lang):

"""Factory method for choosing the style classifier."""

cls = lang == "Perl" and PerlClassifier or UDLClassifier

return cls()

# Parse Perl code

showWarnings = False

class ModuleInfo:

def __init__(self, provide_full_docs):

self.provide_full_docs = provide_full_docs

self.modules = {}

self.currentFunction = None

self.currentNS = None

self.textWrapper = textwrap.TextWrapper()

self.textWrapper.width = 60

self.max_doclet_low_water_mark = 80

self.max_doclet_high_water_mark = 100

self.pod_escape_seq = {'lt': "&lt;",

'gt': "&gt;",

'verbar': "|",

'sol': "/"}

# Things for attrs, etc.

self.export_string = '__exported__'

self.export_ok_string = '__exportable__'

self.local_string = '__local__'

# Cached regular expressions

self.re_bl = r'\r?\n\s*\r?\n'

self.tryGettingDoc_Sig_re3 = re_compile(

r'^=(?:item|head)\w*\s*((?:(?!\n=).)*)(?!\n=)',

re.M | re.S)

self.printDocInfo_re4 = re_compile(

r'^=(?:item|head)\w*\s*((?:(?!\n=).)*)(?!\n=)', re.S | re.M)

self.printDocInfo_re2 = re_compile(

r'^=\w+\s+DESCRIPTION%s(.*?)(?:%s|^=)' % (self.re_bl, self.re_bl), re.M)

self.printDocInfo_re6 = re_compile(

r'^=\w+\s+SYNOPSIS' + self.re_bl + '(.*?)^=',

re.M | re.S)

self.printDocInfo_bdot_re = re_compile(r'\.\s+[A-Z].*\Z')

self._get_first_sentence_re1 = re_compile(r'\.\s+[A-Z].*\Z', re.S)

self._simple_depod_e_re = re_compile(r'E<(.*?)>', re.S)

self._simple_depod_c_re = re_compile(r'C<{2,}\s*(.*?)\s*>{2,}', re.S)

self._simple_depod_ibcfsxl_re1 = re_compile(r'[IBCFSXL]<[^>\n]*>')

self._simple_depod_ibcfsxl_re2 = re_compile(

r'[IBCFSX]<(<*[^<>]*?>*)>', re.S)

self._simple_depod_l_re = re_compile(r'L<\/?(.*?)>')

self._simple_depod_rest_re = re_compile(r'\w<\/?(<*.*?>*)>')

self._depod_re1 = re_compile(

r'^=begin\s+man\s+.*?^=end\s+man\s*', re.M | re.S)

self._depod_re2 = re_compile(r'^=\w+\s*', re.M)

self._depod_re3 = re_compile(r'\]\]>')

self._depod_re4 = re_compile(r'[\x00-\x08\x0b\x0c\x0e-\x1f]')

self.trim_ws_re1 = re_compile(r'(?<=\w[\.\!\?])\s+')

self.trim_ws_re2 = re_compile(r'[\r\n\t]')

self.trim_ws_re3 = re_compile(r' {2,}')

self.printFunctions_re1 = re_compile(r'(\S)\s*\n(?:\s*\n)*\s*(\S)')

def doStartNS(self, ns):

name = ns.name

if name not in self.modules:

self.modules[name] = ns

self.currentNS = ns

def doEndNS(self, **attrInfo):

if 'lineNo' in attrInfo:

self.currentNS.lineend = attrInfo['lineNo']

self.currentNS = None

def getNS(self, name, **attrInfo):

if name in self.modules:

return self.modules[name]

else:

return NamespaceInfo(name, **attrInfo)

def doSetArg(self, name):

self.currentFunction.aArg[name] = []

self.currentFunction.argList.append(name)

def doSetParent(self, **attrInfo):

ns = attrInfo.get('ns')

if ns:

self.currentNS.aParent.append(ns)

def doStartFn(self, fn):

self.currentFunction = fn

def doEndFn(self, **attrInfo):

if 'lineNo' in attrInfo:

self.currentFunction.lineend = attrInfo.get('lineNo')

self.currentNS.aFunc.append(self.currentFunction)

self.currentFunction = None

def doStartVar(self, **attrInfo):

self.thisVar = {}

self.thisVar['name'] = attrInfo.get('name')

for field in ['line', 'aType', 'scope']:

if field in attrInfo:

self.thisVar[field] = attrInfo[field]

def doEndVar(self, forceGlobal):

name = self.thisVar['name']

if (not forceGlobal) and self.currentFunction:

if name in self.currentFunction.aArg:

self.currentFunction.aArg[name].append(self.thisVar)

else:

self.set_or_append(

self.currentFunction.aVar, name, self.thisVar)

else:

self.set_or_append(self.currentNS.aVar, name, self.thisVar)

del self.thisVar

def set_or_append(self, obj, name, val):

if name in obj:

obj[name].append(val)

else:

obj[name] = [val]

def doSetVar(self, **args):

if 'forceGlobal' in args:

forceGlobal = args['forceGlobal']

del args['forceGlobal']

else:

forceGlobal = False

self.doStartVar(**args)

self.doEndVar(forceGlobal)

def add_imported_module(self, args, **kwargs):

args2 = copy.copy(args)

args2.update(kwargs)

if self.currentFunction:

self.currentFunction.aImports.append(args2)

else:

self.currentNS.aImports.append(args2)

def printDocInfo(self, modInfo, funcInfo, currNode):

docs = modInfo.hDocs['modules']

modName = modInfo.name

# These REs need rebuilding each time, as their values change on each

# call.

printDocInfo_re1 = re_compile(r'^=\w+\s+NAME%s%s[\s-]+(.*?)(?:%s|^=)' %

(self.re_bl, modName, self.re_bl), re.M)

try:

mainDocs = self.modules['main'].hDocs['modules'] or []

except:

mainDocs = []

finalDoc = None

if not funcInfo:

# Just dump the module-level docs for now,

# but favor extracting the synopsis.

#

# First, find the first item with a synopsis

#

# Otherwise, go with the first one.

for doc in docs:

m1 = printDocInfo_re1.search(doc)

if m1:

finalDoc = self.trim_ws(m1.group(1), True)

break

else:

m2 = self.printDocInfo_re2.search(doc)

if m2:

finalDoc = self.trim_ws(m2.group(1), True)

break

if finalDoc is None:

# Look only for a qualified name in the NAME section,

# but don't look at the DESCRIPTION part until we can

# select the main class in a module.

for doc in mainDocs:

m1 = printDocInfo_re1.search(doc)

if m1:

finalDoc = self.trim_ws(m1.group(1), True)

break

else:

# First look for the function name in an item

# Then look in the synopsis.

# Look for a top-level synopsis first

funcName = funcInfo.name

# Allow for datasection-based POD\

if docs:

re3_s = (r'''

((?:^=(?:item|head)\w*\s*\r?\n(?:\s*\r?\n)*

.*?(?:%s\s*::|%s\s*->|\$[\w_]+\s*->|[ \t]+)

%s(?![\w\d_]).*\r?\n\s*\r?\n)+)

# Now the description

# Everything up to the an equal-sign (or end)

((?:\r?\n|.)*?)(?:^=|\Z)''' %

(modName, modName, funcName))

printDocInfo_re3 = re_compile(re3_s, re.X | re.M)

for doc in docs:

# Speed up: do index before doing a reg

if doc.find(funcName) == -1:

continue

# Find a function definition in an item or head thing

# Very general-purpose, might pick up false-positives

# The gist:

# Look for one or more =item/head lines,

# separate by blank lines,

# followed by a paragraph description

m1 = printDocInfo_re3.search(doc)

if m1:

part1 = m1.group(1)

finalDoc = self.trim_ws(m1.group(2), True)

part2 = [self.trim_ws(

s, True) for s in self.printDocInfo_re4.findall(part1)]

finalDoc = "\n\n".join(part2) + "\n\n" + finalDoc

finalDoc = self._get_first_sentence(finalDoc)

break

if not finalDoc:

# Look in the __END__ section

# XXXX VERY SLOW

re1_s = r'''(?:^(?:item|head)\w*\s*

.*?

\b%s.*%s)+

# Now the description

# Everything up to the an equal-sign (or end)

((?:\r?\n|.)*?)(?=$|^=)''' % (funcName, self.re_bl)

printDocInfo_re1 = re_compile(re1_s, re.M | re.X)

for doc in mainDocs:

m1 = printDocInfo_re1.search(doc)

if m1:

before_period = self.printDocInfo_bdot_re.sub(

'.', m1.group(1))

finalDoc = self.trim_ws(

self._get_first_sentence(m1.group(1)), True)

break

if not finalDoc:

# Try to find a synopsis entry

printDocInfo_re7 = re_compile(

r'(.*(?:::|->|\b)%s\b.*)' % (funcName,))

for doc in docs + mainDocs:

m1 = self.printDocInfo_re6.search(doc)

if m1:

synopsis = m1.group(1)

m2 = printDocInfo_re7.search(synopsis)

if m2:

finalDoc = self.trim_ws(

self._get_first_sentence(m2.group(1)))

break

if finalDoc:

self.printDocString(finalDoc, currNode)

def _get_first_sentence(self, s1):

s2 = self._get_first_sentence_re1.sub('.', s1)

return s2

def printDocString(self, finalDoc, currNode):

finalDoc2 = self._depod(finalDoc)

if finalDoc2:

currNode.set('doc', finalDoc2)

def _process_e_pod(self, src):

val = self.pod_escape_seq.get(src.lower())

if val:

return val

if re.search(r'^\d+$', src):

return '&#%s;' % src

m1 = re.search(r'^0[Xx]([0-9a-fA-F]+)$', src)

if m1:

return '&#x%s;' % src

else:

# Assume it's an entity name of somekind,

# and return it as an escaped representation

# For example, pod E<eacute> will be encoded as

# &amp;eacute;

# and will appear as

# &eacute;

# Not great, but it causes no breakage.

return '&amp;%s' % src

def _wrap_process_e_pod(self, m):

return self._process_e_pod(m.group(1))

def _simple_depod(self, doc):

# Simple inline-markup removal (doesn't handle nested inlines)

# In Perl, do this:

# $doc =~ s/E<(.*?)>/_process_e_pod($1)/eg;

doc = re_sub(self._simple_depod_e_re, self._wrap_process_e_pod, doc)

# And handle the inline codes-- thse nest with E codes...

doc = self._simple_depod_c_re.sub(r'\1', doc)

# Above code replaces this:

# doc = re_sub(r'C<{2,}\s+(.*?)\s+>{2,}', r'\1', doc)

# doc = XmlAttrEscape(doc) -- No longer needed with ElementTree

# Allow the other sequences to nest, and loop until there

# aren't any left.

old_doc = doc

while self._simple_depod_ibcfsxl_re1.search(doc):

# Most formatting sequences wrap a single clump of code

doc = self._simple_depod_ibcfsxl_re2.sub(r'\1', doc)

# Handling of links - this is more complicated.

doc = self._simple_depod_l_re.sub(r'\1', doc)

# We need to make sure we pull out when nothing changes.

# XXX A log message would be useful here.

if old_doc != doc:

old_doc = doc

else:

break

# Remove any unrecognized sequences

doc = self._simple_depod_rest_re.sub(r'\1', doc)

# And shrink entities back into strings.

# This is done because we can't convert constructs like

# E<gt> into ">" directly, because they'll prevent

# proper handling of outer C<...> in strings like 'C<if (a E<gt> b) {

# ...>'

doc = doc.replace("&lt;", "<").replace("&gt;", ">")

return doc

# Precondition: this text is emitted inside a cdata-section

def _depod(self, doc):

# Remove embedded man directives

doc1 = self._depod_re1.sub('', doc)

# Pull out leading equal signs and the directives

doc2 = self._depod_re2.sub('', doc1)

doc3 = self._simple_depod(doc2)

# Handle strings that could cause the XML parser trouble

doc4 = self._depod_re3.sub(']<!>]>', doc3)

doc5 = self._depod_re4.sub('?', doc4)

return doc5

def trim_ws(self, str1, truncate=False):

# First split into sentences

str2 = str1.strip()

if truncate:

sentences = self.trim_ws_re1.split(str2)

keep_sentences = []

sum = 0

# Keep sentences until we pass max_doclet_high_water_mark chars.

for s in sentences:

thisLen = len(s)

if sum and sum + thisLen > self.max_doclet_high_water_mark:

break

s1 = self.trim_ws_re2.sub(' ', s)

s2 = self.trim_ws_re3.sub(' ', s1)

keep_sentences.append(s2)

sum += thisLen

if sum > self.max_doclet_high_water_mark:

break

str2 = " ".join(keep_sentences)

if str2.find("\n") >= 0 or len(str2) > self.textWrapper.width * 1.1:

str2 = "\n".join(self.textWrapper.wrap(str2))

return str2

def printClassParents(self, modInfo, currNode):

if not hasattr(modInfo, 'aParent'):

return

classrefs = [info[0] for info in modInfo.aParent]

if len(classrefs) > 0:

currNode.set('classrefs', " ".join(classrefs))

def printImports(self, modInfo, currNode):

# -- this will be correct only when there are deliberate conflicts

# better to use object inheritance to choose methods dynamically.

# imports = getattr(modInfo, 'aImports', [])

# imports.reverse()

for _import in getattr(modInfo, 'aImports', []):

attrs = list(_import.keys())

importNode = SubElement(currNode, "import")

for k in attrs:

importNode.set(k, str(_import[k]))

def printTypeInfo(self, argInfo, currNode):

types = {}

for type_ in argInfo:

typeInfo = type_.get('aType')

if not typeInfo:

continue

elif 'assign' not in typeInfo:

continue

tp = typeInfo['assign']

if tp in types:

continue

types[tp] = None

currNode.set('citdl', tp)

def printVariables(self, modInfo, currNode):

if not hasattr(modInfo, 'aVar'):

return

def sorter1(a):

return (a[0]['line'], a[0]['name'].lower())

variables = list(modInfo.aVar.values())

variables.sort(key=sorter1)

try:

export_info = modInfo.export_info

except:

if not hasattr(self, 'default_export_info'):

self.default_export_info = {'@EXPORT': {}, '@EXPORT_OK': {}}

export_info = self.default_export_info

for varInfo in variables:

var_name = varInfo[0]['name']

varNode = SubElement(

currNode, 'variable', line=str(varInfo[0]['line']),

name=var_name)

attr_parts = []

if var_name in export_info['@EXPORT']:

attr_parts.append(self.export_string)

if var_name in export_info['@EXPORT_OK']:

attr_parts.append(self.export_ok_string)

try:

if varInfo[0]['scope'] == 'my':

attr_parts.append(self.local_string)

except:

pass

if attr_parts:

varNode.set('attributes', ' '.join(attr_parts))

self.printTypeInfo(varInfo, varNode)

def printFunctions(self, modInfo, currNode):

for funcInfo in getattr(modInfo, 'aFunc', []):

sig, docString = self.tryGettingDoc_Sig(modInfo, funcInfo)

funcName = funcInfo.name

if not sig:

sig = '%s(%s)' % (funcName, ', '.join(funcInfo.argList))

else:

sig = self._simple_depod(sig.strip())

sig = self.printFunctions_re1.sub('\\1\n\\2', sig)

funcNode = SubElement(

currNode, 'scope', ilk='function', name=funcName)

for attr_name in ['line', 'lineend']:

ln = getattr(funcInfo, attr_name, None)

if ln:

funcNode.set(attr_name, str(ln))

attr_parts = []

if funcInfo.isConstructor:

attr_parts.append("__ctor__")

export_info = modInfo.export_info

for tuple in [['@EXPORT', self.export_string],

['@EXPORT_OK', self.export_ok_string]]:

if funcName in export_info[tuple[0]]:

attr_parts.append(tuple[1])

if attr_parts:

funcNode.set('attributes', ' '.join(attr_parts))

funcNode.set('signature', sig)

for argName in funcInfo.argList:

argInfo = funcInfo.aArg.get(argName)

if argInfo:

kwargs = {'ilk': 'argument', 'name': argInfo[0]['name']}

if 'line' in argInfo[0]:

kwargs['line'] = str(argInfo[0]['line'])

argNode = SubElement(funcNode, 'variable', **kwargs)

self.printTypeInfo(argInfo, argNode)

if self.provide_full_docs:

if not docString:

self.printDocInfo(modInfo, funcInfo, funcNode)

else:

self.printDocString(docString, funcNode)

self.printImports(funcInfo, funcNode)

self.printVariables(funcInfo, funcNode)

def tryGettingDoc_Sig(self, modInfo, funcInfo):

if not self.provide_full_docs:

return (None, None)

modName = modInfo.name

funcName = funcInfo.name

re1_s = r"""((?:^=(?:item|head)\w*\s*

.*?(?:%s\s*\:\:|%s\s*->|\$[\w_]+\s*->|[ \t])

%s(?:[^\w_]|$).*%s)+)

# Now the description

# Everything up to the an equal-sign (or end)

((?:\r?\n|.)*?)(?:^=|\Z)""" % \

(modName, modName, funcName, self.re_bl)

re1 = re_compile(re1_s, re.X | re.M)

re2a = re_compile(funcName + r'\s+\w')

re2b = re_compile(r'\s\w+\s+' + funcName + r'\b')

re4_s = (r'''^=(?:item|head)\s*\*?\s*\r?\n

(?:^\s*\r?\n)*

^\s*C<+(.*%s.*)>+\s*\r?\n

(?:^\s*\r?\n)*

# Now the description

# Everything up to the an equal-sign (or end)

((?:.*\r?\n(?!=))+)''' % (funcName,))

re4 = re_compile(re4_s, re.M | re.X)

for doc in modInfo.hDocs['modules'] + self.modules['main'].hDocs['modules']:

if doc.find(funcName) == -1:

continue

m1 = re1.search(doc)

if m1:

part1 = m1.group(1)

# If the function name is followed by text, it's probably

# an English description. We need to check for a whole word

# before the name to avoid the pod directive

if (re2a.search(part1) or

re2b.search(part1)):

continue

finalDoc = self.trim_ws(

self._get_first_sentence(m1.group(2)), True)

part2 = [

s.strip() for s in self.tryGettingDoc_Sig_re3.findall(part1)]

finalSig = (part2 and "\n\n".join(part2)) or None

return (finalSig, finalDoc)

else:

m1 = re4.search(doc)

if m1:

finalSig = m1.group(1)

finalDoc = self.trim_ws(

self._get_first_sentence(m1.group(2)), True)

return (finalSig, finalDoc)

return (None, None)

class NamespaceInfo:

def __init__(self, name, **attrInfo):

self.name = name

self.line = attrInfo.get('lineNo') or 0

self.aFunc = []

self.aVar = {}

self.aParent = []

self.hDocs = {'modules': [], # hash of modules => array of docs,

'subs': {} # subs => subname => array of docs

}

self.aImports = []

self._isProbablyClass = False

self.export_info = {'@EXPORT': {},

'@EXPORT_OK': {}}

def isProbablyClass(self, val):

self._isProbablyClass = val

class FunctionInfo:

def __init__(self, name, **attrInfo):

self.name = name

self.aArg = {}

self.aVar = {}

self.resultType = []

self.argList = []

self.aImports = []

self.isConstructor = attrInfo.get('isConstructor', False)

if 'lineNo' in attrInfo:

self.line = attrInfo.get('lineNo')

if not os.path.altsep or os.path.altsep == os.path.sep:

def pathSplitter(s):

return s.split(os.path.sep)

else:

def pathSplitter(s):

return re.split(re_compile('[' + re.escape(os.path.sep)

+ re.escape(os.path.altsep) + ']'), s)

class Parser:

def __init__(self, tokenizer, lang="Perl", provide_full_docs=True):

self.tokenizer = tokenizer

self._provide_full_docs = provide_full_docs

self.block_stack = []

self.bracket_matchers = {"[": "]", "{": "}", "(": ")"}

self.classifier = _get_classifier(lang)

# Use simple knowledge of Perl's syntax

# to skip quickly through code to skip.

self.opHash = {"(": [0, 1],

")": [0, -1],

"{": [1, 1],

"}": [1, -1],

"[": [2, 1],

"]": [2, -1]}

self.pragmaNames = {'attributes': None,

'attrs': None,

'autouse': None,

'bigint': None,

'bignum': None,

'bigrat': None,

'blib': None,

'bytes': None,

'charnames': None,

'constant': None,

'diagnostics': None,

'encoding': None,

'fields': None,

'filetest': None,

'if': None,

'integer': None,

'less': None,

'lib': None,

'locale': None,

'open': None,

'ops': None,

'overload': None,

're': None,

'sigtrap': None,

'sort': None,

'strict': None,

'subs': None,

'threads': None,

'utf8': None,

'vmsish': None,

'warnings': None,

}

self.find_open_indexer_re = re_compile(r'[\[{]')

self.provide_full_docs = provide_full_docs

def _is_stmt_end_op(self, tok):

tval = tok['text']

if self.classifier.is_keyword(tok, 'or'):

return True

elif self.classifier.is_any_operator(tok):

return tval in (';', '||')

return False

def _is_string(self, tok):

return tok['style'] in self.tokenizer.string_types

def printHeader(self, mtime):

moduleName = self.moduleName

root = Element("codeintel", version="2.0")

fileNode = Element("file", lang="Perl",

path=moduleName)

if mtime:

fileNode.set('mtime', str(mtime))

root.append(fileNode)

return (root, fileNode)

def printContents(self, moduleContentsName, currNode):

name = os.path.splitext(os.path.basename(self.moduleName))[0]

moduleNode = SubElement(

currNode, 'scope', ilk='blob', lang="Perl", name=name)

innerModules = self.moduleInfo.modules

mainInfo = innerModules.get('main', None)

if mainInfo:

if self.provide_full_docs:

self.moduleInfo.printDocInfo(mainInfo, None, moduleNode)

self.moduleInfo.printImports(mainInfo, moduleNode)

self.moduleInfo.printVariables(mainInfo, moduleNode)

def sorter1(a):

amod = innerModules.get(a)

return getattr(amod, 'line', None), getattr(amod, 'name', "")

# Sub-packages need to updated their parent blob name - bug 88814.

# I.e. when parsing "XML/Simple.pm" the blob name is "Simple", but we

# need it be "XML::Simple" in this case. The bestPackageName is

# used to find the best matching name.

packages = [x for x in list(self.moduleInfo.modules.keys()) if x != 'main']

packages.sort(key=sorter1)

bestPackageName = None

for k in packages:

modInfo = innerModules[k]

if name in k and \

(bestPackageName is None or len(bestPackageName) > k):

bestPackageName = k

moduleNode.set("name", bestPackageName)

classNode = SubElement(

moduleNode, 'scope', ilk='class', name=modInfo.name, line=str(modInfo.line))

if hasattr(modInfo, 'lineend'):

classNode.set('lineend', str(modInfo.lineend))

self.moduleInfo.printClassParents(modInfo, classNode)

if self.provide_full_docs:

self.moduleInfo.printDocInfo(modInfo, None, classNode)

self.moduleInfo.printImports(modInfo, classNode)

self.moduleInfo.printVariables(modInfo, classNode)

self.moduleInfo.printFunctions(modInfo, classNode)

# And do main's functions after its classes

if mainInfo:

self.moduleInfo.printFunctions(mainInfo, moduleNode)

def printTrailer(self, root):

pass

def at_end_expression(self, tok):

if not self.classifier.is_any_operator(tok):

return False

return tok['text'] in (';', ',', '}')

def collect_multiple_args(self, origLineNo, context, var_scope):

nameList = []