#!/usr/bin/env python

# ***** BEGIN LICENSE BLOCK *****

# Version: MPL 1.1/GPL 2.0/LGPL 2.1

#

# The contents of this file are subject to the Mozilla Public License

# Version 1.1 (the "License"); you may not use this file except in

# compliance with the License. You may obtain a copy of the License at

# http://www.mozilla.org/MPL/

#

# Software distributed under the License is distributed on an "AS IS"

# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the

# License for the specific language governing rights and limitations

# under the License.

#

# The Original Code is Komodo code.

#

# The Initial Developer of the Original Code is ActiveState Software Inc.

# Portions created by ActiveState Software Inc are Copyright (C) 2000-2007

# ActiveState Software Inc. All Rights Reserved.

#

# Contributor(s):

# ActiveState Software Inc

#

# Alternatively, the contents of this file may be used under the terms of

# either the GNU General Public License Version 2 or later (the "GPL"), or

# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),

# in which case the provisions of the GPL or the LGPL are applicable instead

# of those above. If you wish to allow use of your version of this file only

# under the terms of either the GPL or the LGPL, and not to allow others to

# use your version of this file under the terms of the MPL, indicate your

# decision by deleting the provisions above and replace them with the notice

# and other provisions required by the GPL or the LGPL. If you do not delete

# the provisions above, a recipient may use your version of this file under

# the terms of any one of the MPL, the GPL or the LGPL.

#

# ***** END LICENSE BLOCK *****

"""New citree-based codeintel evaluation engine.

A 'citree' is basically an ElementTree of a CIX document with some tweaks.

The idea is to use these for completion/calltip evaluation instead of the

CIDB. This is mainly about performance but also about fixing some

limitations, bugs, and having a better code design (i.e. where lang-specific

quirks can be dealt with cleanly).

"""

import sys

from os.path import normpath

import logging

import re

import ciElementTree as ET

if not getattr(ET, "_patched_for_komodo_", False):

import warnings

warnings.warn("Not using codeintel's patched elementtree: "

"this may cause problems")

from codeintel2.common import *

from codeintel2.citadel import CitadelEvaluator

log = logging.getLogger("codeintel.tree")

CIX_VERSION = "2.0"

def tree_2_0_from_tree_0_1(tree):

"""Convert CIX 0.1 to CIX 2.0."""

# - update some of the no longer used <file> attributes

# - drop "generator"

try:

del tree[0].attrib["generator"]

except KeyError:

pass

# - drop 'md5' and 'mtime' on the <file> tag

try:

del tree[0].attrib["md5"]

except KeyError:

pass

try:

del tree[0].attrib["mtime"]

except KeyError:

pass

# - move "language" attribute on <file> to "lang" and to "lang" on

# <module> (New multi-lang CIX output will allow one file to

# have modules of different langs.)

for file in tree.getiterator("file"):

lang = file.get("language")

if lang is not None:

file.set("lang", lang)

for module in file.getiterator("module"):

if module.get("lang") is None:

module.set("lang", lang)

try:

del file.attrib["language"]

except KeyError:

# Be tolerant of transitional CIX.

pass

# - move <doc> and <signature> optional sub tags into parent

# attribute

# PERF: This could be done better.

for tag in ("variable", "function", "class", "module", "interface",

"argument", "classref", "interfaceref"):

for node in tree.getiterator(tag):

for child in reversed(node):

# reversed() so can modify while iterating over

if child.tag == "signature":

if child.text: # be tolerant of <signature />

node.set("signature", child.text)

node.remove(child)

elif child.tag == "doc":

if child.text: # be tolerant of <doc />

node.set("doc", child.text)

node.remove(child)

if not node: # no children now

node.text = None

# - move non-variable tags to attributes

# (XXX currently <classref> and <interfaceref> tags are not moved)

for tag in ("variable", "argument", "classref", "interfaceref"):

for node in tree.getiterator(tag):

for child in reversed(node):

if child.tag == "type":

node.set("citdl", child.get("type"))

node.remove(child)

if not node: # no remaining children

node.text = None

if tag == "argument":

node.tag = "variable"

node.set("ilk", "argument")

# - move <returns> to a <function> attribute

for node in tree.getiterator("function"):

for child in reversed(node): # PERF: could just check last child

if child.tag == "returns":

assert child[0].tag == "type"

node.set("returns", child[0].get("type"))

node.remove(child)

# - move classrefs and interfacerefs to attributes

# Note: <classref attribute="__mixin__"> => "mixinrefs" attribute.

# This is used by Ruby (though not used for eval, yet).

for scope_ilk in ("class", "interface"):

for node in tree.getiterator(scope_ilk):

interfacerefs = []

classrefs = []

mixinrefs = []

for child in reversed(node):

if child.tag == "classref":

if "__mixin__" in child.get("attributes", ""):

mixinrefs.append(child.get("citdl")

or child.attrib["name"])

else:

classrefs.append(child.get("citdl")

or child.attrib["name"])

node.remove(child)

elif child.tag == "interfaceref":

interfacerefs.append(child.get("citdl")

or child.attrib["name"])

node.remove(child)

if classrefs:

classrefs.reverse()

assert not [c for c in classrefs if ' ' in c]

node.set("classrefs", ' '.join(classrefs))

if interfacerefs:

interfacerefs.reverse()

assert not [i for i in interfacerefs if ' ' in i]

node.set("interfacerefs", ' '.join(interfacerefs))

if mixinrefs:

mixinrefs.reverse()

assert not [m for m in mixinrefs if ' ' in m]

node.set("mixinrefs", ' '.join(mixinrefs))

if len(node) == 0:

node.text = None

# - make all scope tags a "scope" tag (easier for elem.find() usage)

for tag in ("class", "function", "interface", "module"):

for node in tree.getiterator(tag):

node.tag = "scope"

if tag == "class" and "__namespace__" in node.get("attributes", ""):

node.set("ilk", "namespace")

attributes = node.get("attributes").split()

attributes.remove("__namespace__")

if not attributes:

del node.attrib["attributes"]

else:

node.set("attributes", ' '.join(attributes))

elif tag == "module":

node.set("ilk", "blob")

else:

node.set("ilk", tag)

tree.set("version", "2.0")

return tree

def tree_from_cix_path(cix_path):

"""Return a (ci)tree for the CIX content in the given path.

Raises pyexpat.ExpatError if the CIX content could not be parsed.

"""

tree = ET.parse(cix_path).getroot()

version = tree.get("version")

if version == CIX_VERSION:

return tree

elif version == "0.1":

return tree_2_0_from_tree_0_1(tree)

else:

raise CodeIntelError("unknown CIX version: %r" % version)

def tree_from_cix(cix):

"""Return a (ci)tree for the given CIX content.

Raises pyexpat.ExpatError if the CIX content could not be parsed.

"""

if isinstance(cix, str):

cix = cix.encode("UTF-8", "xmlcharrefreplace")

tree = ET.XML(cix)

version = tree.get("version")

if version == CIX_VERSION:

return tree

elif version == "0.1":

return tree_2_0_from_tree_0_1(tree)

else:

raise CodeIntelError("unknown CIX version: %r" % version)

def pretty_tree_from_tree(tree, indent_width=2):

"""Add appropriate .tail and .text values to the given tree so that

it will have a pretty serialization.

Note: This modifies the tree *in-place*.

Presumption: This is a CIX 2.0 tree.

"""

INDENT = ' '*indent_width

def _prettify(elem, indent_level=0):

if elem: # i.e. has children

elem.text = '\n' + INDENT*(indent_level+1)

for child in elem:

_prettify(child, indent_level+1)

elem[-1].tail = '\n' + INDENT*indent_level

elem.tail = '\n' + INDENT*indent_level

else:

elem.text = None

elem.tail = '\n' + INDENT*indent_level

_prettify(tree)

return tree

def check_tree(tree):

"""Generate warnings/errors for common mistakes in CIX trees.

Yields tuples of the form:

("warning|error", <msg>)

"""

assert tree.tag == "codeintel",\

"can only check starting from <codeintel> element"

assert tree.get("version") == CIX_VERSION, \

"can only check CIX v%s trees" % CIX_VERSION

# - file 'lang' is set, not 'language'

file = tree[0]

if not file.get("lang"):

yield ("error", "no 'lang' attr on <file> element")

if file.get("language"):

yield ("warning", "'language' attr on <file> element is obsolete,"

"use 'lang'")

for blob in file:

if blob.get("ilk") != "blob":

yield ("error", "element under <file> is not ilk=blob: %r" % blob)

# - blob 'lang' is set

if not blob.get("lang"):

yield ("error", "no 'lang' attr on <blob> element: %r" % blob)

# - classrefs are space separated, not with commas (warn)

for class_elem in blob.getiterator("scope"):

if class_elem.get("ilk") != "class":

continue

classrefs = class_elem.get("classrefs")

if not classrefs:

continue

if ',' in classrefs:

yield ("warning", "multiple class references in 'classrefs' "

"attr on class scopes must be "

"space-separated: %r may be using "

"comma-separation: %r"

% (class_elem, classrefs))

class TreeEvaluator(CitadelEvaluator):

def get_start_scoperef(self):

linenum = self.line + 1 # convert to 1-based

try:

blob = self.buf.blob_from_lang[self.trg.lang]

except KeyError:

raise EvalError("no %s scan info for %r" % (self.lang, self.buf))

return self.buf.scoperef_from_blob_and_line(blob, linenum)

def eval(self, mgr):

self.mgr = mgr

self.citadel = mgr.citadel

if self.ctlr.is_aborted():

self.ctlr.done("aborting")

return

self.ctlr.info("eval %s %s", self, self.trg)

self.pre_eval()

try:

if self.trg.form == TRG_FORM_CPLN:

cplns = self.eval_cplns()

if cplns:

cplns = self.post_process_cplns(cplns)

self.info(" cplns: %r", cplns)

if cplns:

self.ctlr.set_cplns(cplns)

elif self.trg.form == TRG_FORM_CALLTIP:

calltips = self.eval_calltips()

if calltips:

calltips = self.post_process_calltips(calltips)

self.info(" calltips: %r", calltips)

if calltips:

self.ctlr.set_calltips(calltips)

else: # self.trg.form == TRG_FORM_DEFN

defns = self.eval_defns()

if defns:

defns = Definition.unique_definitions(defns)

defns = self.post_process_defns(defns)

self.info(" defns: %r", defns)

if defns:

self.ctlr.set_defns(defns)

self.ctlr.done("success")

except CodeIntelError as ex:

# XXX Should we have an error handling hook here?

self.ctlr.error("evaluating %s: %s", self, ex)

self.ctlr.done("eval error")

except Exception:

log.exception("Unexpected error with evaluator: %s", self)

# Must still mark done on the ctlr to avoid leaks - bug 65502.

self.ctlr.done("eval error")

def scope_stack_from_tree_and_linenum(self, tree, linenum):

"""Get the start scope for the given line.

"linenum" appears to be 0-based, however all CIX line data

is 1-based so we'll convert here.

Dev Notes:

- XXX Add built-in scope.

"""

linenum += 1 # convert to 1-based

# XXX This is presuming that the tree has only one blob.

scope_stack = [tree.find("file/scope")]

while True:

next_scope_could_be = None

# PERF: Could make this a binary search if a scope has *lots* of

# subscopes.

for scope in scope_stack[-1].findall("scope"):

start = int(scope.get("line"))

if start <= linenum \

and (not scope.get("lineend")

or linenum <= int(scope.get("lineend"))):

next_scope_could_be = scope

elif start > linenum:

break

if next_scope_could_be is not None:

scope_stack.append(next_scope_could_be)

else:

break

return scope_stack

# TODO: split out '()' as a separate token.

def _tokenize_citdl_expr(self, citdl):

for token in citdl.split('.'):

yield token

def _join_citdl_expr(self, tokens):

return '.'.join(tokens)

def str_elem(self, elem):

if elem.tag == "scope":

return "%s %s" % (elem.get("ilk"), elem.get("name"))

else:

return "%s %s" % (elem.tag, elem.get("name"))

def str_elem_and_children(self, elem):

s = [self.str_elem(elem)]

for child in elem:

s.append(self.str_elem(child))

return "%s: %s" % (self.str_elem(elem),

', '.join(self.str_elem(c) for c in elem))

def str_import(self, elem):

# c.f. cb.py::getDescForImport()

module = elem.get("module")

symbol = elem.get("symbol")

alias = elem.get("alias")

if alias and symbol:

s = "from %(module)s import %(symbol)s as %(alias)s" % locals()

elif alias:

s = "import %(module)s as %(alias)s" % locals()

elif symbol:

s = "from %(module)s import %(symbol)s" % locals()

else:

s = "import %(module)s" % locals()

return s

# logging funcs (perhaps best on controller)

def log_start(self):

self._log = []

def log(self, msg, *args, **kwargs):

"""

kwargs:

"cached" (boolean) indicates if result was from cache

"""

log_indent = ' '*4

if True: # just print as we go

if args:

s = [msg % args]

else:

s = [msg]

if kwargs.get("cached"):

s.append(" (cached)")

self.info('%s', ''.join(s))

else: # capture log for latter printing

self._log.append(msg, args, kwargs)

def pre_eval(self):

self.curr_tree = self.buf.tree

# ET.dump(self.curr_tree)

def _eval_citdl_expr(self, expr, scope_stack):

"""Return the citree node for the given CITDL expression.

os.environ.get() -> <class 'str' on stdlib blob 'built-in'>

"""

tokens = list(self._tokenize_citdl_expr(expr))

assert tokens, "have to adjust handling if no tokens"

obj = self._eval_citdl_token(tokens[0], scope_stack)

for i, token in enumerate(tokens[1:]):

if token.endswith("()"):

token = token[-2:]

call = True

else:

call = False

if obj.tag == "import":

obj = self._eval_import_getattr(obj, token,

self._join_citdl_expr(tokens[:i+2]))

else:

# XXX marky: this code does not appear to be used!

# (nobody seems to define _eval_getattr)

obj = self._eval_getattr(obj, token,

self._join_citdl_expr(tokens[:i+2]))

if call:

raise CodeIntelError("_eval_citdl_expr(%r): not handling "

"call on %r "

% (expr, self.str_elem(obj)))

if obj.tag == "import":

raise CodeIntelError("_eval_citdl_expr: cannot return import "

"<%s>: need to resolve it"

% self.str_import(obj))

return obj

def _resolve_import(self, module_name, symbol_name=None):

"""Return a loaded citree node for the given import info.

"module_name" is the name of the module to import.

"symbol_name" (if given) is the attribute on that module to

return.

"""

# TODO: get logging right

# XXX All the logging stuff should go on the controller and that

# should get passed in here for appropriate logging of this

# eval.

# XXX Will this screw up for, e.g. in Python:

# 'import codeintel.utils'?

import_handler = self.citadel.import_handler_from_lang(self.lang)

module = import_handler.import_blob_name(

module_name, self.buf.libs, self.ctlr)

self.log("module '%s' imports <%s>", module_name,

self.str_elem(module))

if symbol_name:

# XXX logging won't be right here

return self._eval_getattr(module, symbol_name,

"%s.%s" % (module_name, symbol_name))

# XXX Here is the _eval_getattr code to duplicate.

# self.log("lookup '%s' on <%s>:", name, self.str_elem(obj))

# for child in obj:

# if child.get("name") == name:

# attr = child

# self.log("'%s' is <%s>", citdl_expr, self.str_elem(child))

# return attr

# else:

# raise CodeIntelError("couldn't find '%s' attribute on <%s>"

# % (name, self.str_elem(obj)))

else:

return module

def _eval_import(self, imp, name):

"""Return the object imported, if any, with the given import

node (in a citree) and name.

Return value: If successful it returns the citree node imported.

If 'name' was not found in a '*'-import then None is returned

(e.g. it is not exceptional that 'from os import *' does not

import 'fuzzywuzzy'). If the import could not be resolved, but

it looks like it should have been, then an error is raised.

"""

# One of these:

# 'os' may be from <import os>:

# ...

# 'os' is <blob os>

# 'os' is from <import os>: <blob os> (cached)

#

# Python

# if non-* import and matches:

# 'os' is from <import os>

# is <import os> from <project foo>? no

# ...

# is <import os> from <python-2.4-stdlib>? yes: <blob os>

# 'os' is <blob os>

#

# 'dirname' may be from <from os.path import *>:

# is <from os.path import *> from <project foo>? no

# ...

# is <from os.path import *> from <python-2.4-stdlib>? yes: <blob

# os.path>

# TOTEST:

# - 'from xml import dom', does that get it right? I doubt it.

module_name = imp.get("module")

symbol_name = imp.get("symbol")

alias = imp.get("alias")

obj = None

if alias:

if alias == name: # <import foo as name> or <from foo import bar as name>

self.log("'%s' is from <%s>", name, self.str_import(imp))

return self._resolve_import(module_name, symbol_name)

elif symbol_name:

assert symbol_name != "**", "only Perl should be using '**' for imports"

if symbol_name == "*": # <from foo import *>

self.log("'%s' may be from <%s>", name, imp)

# XXX some variation of _resolve_import to specify just

# importing the module.

try:

module = self._resolve_import(module_name)

except CodeIntelError as ex: # use equivalent of NoModuleEntry?

self.warn("could not resolve '%s' import to handle <%s>",

module_name, self.str_import(imp))

return None

# TODO:

# see if name in export list (__all__ for Python,

# @EXPORT for Perl, default all top-level symbols)

# if so, eval getattr of name on module object

self.warn("not handling <%s>!", self.str_import(imp))

if symbol_name == name: # <from foo import name>

self.log("'%s' is from <%s>", name, self.str_import(imp))

return self._resolve_import(module_name, symbol_name)

elif module_name == name: # <import foo>

self.log("'%s' is from <%s>", name, self.str_import(imp))

return self._resolve_import(module_name)

return None

def _eval_citdl_token(self, token, scope_stack):

start_scope_str = self.str_elem(scope_stack[-1])

self.log("eval '%s' at <%s>:", token, start_scope_str)

while scope_stack:

scope = scope_stack.pop()

self.log("is '%s' accessible on <%s>?",

token, self.str_elem(scope))

for child in reversed(scope):

# Check children in reverse because last definition in

# file wins. A correct refinement *for the top-level*

# would be to skip anything defined later in the file

# than the current start position.

# TODO-PERF: The list of symbols on a scope should be a

# dict to speed up this loop. This is complicated

# by '*' imports.

if child.tag == "import":

obj = self._eval_import(child, token)

if obj:

return obj

elif child.get("name") == token:

obj = child

if obj.tag == "variable":

citdl = obj.get("citdl")

if not citdl:

self.log("'%s' is <%s> which is of unknown type",

token, self.str_elem(obj))

raise CodeIntelError(

"don't know type of <%s> on <%s>"

% (self.str_elem(obj), self.str_elem(scope)))

else:

self.log("'%s' is <%s> which is '%s'", token,

self.str_elem(obj), citdl)

obj = self._eval_citdl_expr(

citdl, scope_stack+[scope])

self.log("'%s' is <%s>", token,

self.str_elem(obj))

else:

self.log("'%s' is <%s>", token, self.str_elem(obj))

return obj

else:

continue

else:

raise CodeIntelError("couldn't resolve '%s' starting at %s"

% (token, start_scope_str))

def _defn_from_hit(self, hit):

elem, (blob, lpath) = hit

# self.log("_defn_from_hit:: blob: %r", blob)

# for attr_name, attr_value in blob.attrib.items():

# self.log("attr_name: %r, attr_value: %r", attr_name, attr_value)

# self.log("_defn_from_hit:: elem: %r", elem)

path = blob.get("src", None)

name = elem.get("name", None)

line = elem.get(

"line", 1) # e.g. for an import, just use the first line

if line is not None:

try:

line = int(line)

except ValueError:

line = 1

ilk = elem.get("ilk") or elem.tag

citdl = elem.get("citdl", None)

doc = elem.get("doc", None)

signature = elem.get("signature", None)

attributes = elem.get("attributes", None)

returns = elem.get("returns", None)

try:

scope = self._elem_from_scoperef((blob, lpath))

except AttributeError:

scopestart = 1

scopeend = 0

else:

def safe_int_get(attr, default_value):

try:

return int(scope.get(attr, default_value))

except (ValueError, AttributeError):

return default_value

scopestart = safe_int_get("line", 1)

scopeend = safe_int_get("lineend", 0)

# Only fixup paths that do not look like URIs.

if path and not re.match(r"^\w+:\/\/", path):

if sys.platform == "win32":

path = path.replace('/', '\\') # unnormalize path

path = normpath(path) # remove possible '.' and '..' elements

defn = Definition(blob.get("lang"), path, blob.get("name"), lpath,

name, line, ilk, citdl, doc,

signature, attributes, returns, scopestart, scopeend)

return defn

class _infinite_recursion_checker(object):

def __init__(self, evalr):

self.evalr = evalr

def __enter__(self):

self.evalr._eval_count_all += 1

if self.evalr._eval_count_all >= self.evalr._SENTINEL_MAX_ALL_COUNT:

raise EvalError("Too much recursion")

def __exit__(self, exc_type, exc_value, traceback):

self.evalr._eval_count_all -= 1

# The SENTINEL_MAX_EXPR_COUNT could probably be *reduced*.

# Note: This is an approximation that we are infinitely looping

# on the same evaluation. The *actual* appropriate key would be:

#

# (expr, scoperef)

#

# but that is overkill for now, I think.

_SENTINEL_MAX_EXPR_COUNT = 10

_SENTINEL_MAX_ALL_COUNT = 100

_eval_count_from_expr = None

_eval_count_all = 0

def _check_infinite_recursion(self, expr):

if self._eval_count_from_expr is None:

# Move this init into eval() when on TreeEvalutor.

self._eval_count_from_expr = {}

eval_count = self._eval_count_from_expr.get(expr, 0)

eval_count += 1

if eval_count >= self._SENTINEL_MAX_EXPR_COUNT:

raise EvalError("hit eval sentinel: expr '%s' eval count "

"is %d (abort)" % (expr, eval_count))

self._eval_count_from_expr[expr] = eval_count

return TreeEvaluator._infinite_recursion_checker(self)

#---- internal support stuff

def _dump_element(elem, indent=''):

"""Dump an element tree without using ET.dump() which

(1) uses XML syntax,

(2) can blow up if an attribute is set to None accidentally.

This is only useful for debugging.

"""

s = "%selement '%s': %s" % (indent, elem.tag, elem.attrib)

print(s)

for child in elem:

_dump_element(child, indent+' ')