#!python

# ***** BEGIN LICENSE BLOCK *****

# Version: MPL 1.1/GPL 2.0/LGPL 2.1

#

# The contents of this file are subject to the Mozilla Public License

# Version 1.1 (the "License"); you may not use this file except in

# compliance with the License. You may obtain a copy of the License at

# http://www.mozilla.org/MPL/

#

# Software distributed under the License is distributed on an "AS IS"

# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the

# License for the specific language governing rights and limitations

# under the License.

#

# The Original Code is Komodo code.

#

# The Initial Developer of the Original Code is ActiveState Software Inc.

# Portions created by ActiveState Software Inc are Copyright (C) 2000-2007

# ActiveState Software Inc. All Rights Reserved.

#

# Contributor(s):

# ActiveState Software Inc

#

# Alternatively, the contents of this file may be used under the terms of

# either the GNU General Public License Version 2 or later (the "GPL"), or

# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),

# in which case the provisions of the GPL or the LGPL are applicable instead

# of those above. If you wish to allow use of your version of this file only

# under the terms of either the GPL or the LGPL, and not to allow others to

# use your version of this file under the terms of the MPL, indicate your

# decision by deleting the provisions above and replace them with the notice

# and other provisions required by the GPL or the LGPL. If you do not delete

# the provisions above, a recipient may use your version of this file under

# the terms of any one of the MPL, the GPL or the LGPL.

#

# ***** END LICENSE BLOCK *****

"""UDL (User-Defined Language) support for codeintel."""

import os

from os.path import dirname, join, abspath, normpath, basename, exists

import sys

import re

import logging

import threading

import operator

import traceback

from pprint import pprint, pformat

import SilverCity

from SilverCity import ScintillaConstants

from SilverCity.ScintillaConstants import * # XXX import only what we need

from SilverCity.Lexer import Lexer

from codeintel2.common import *

from codeintel2.citadel import CitadelBuffer

# from codeintel2.javascript_common import trg_from_pos as

# javascript_trg_from_pos

if _xpcom_:

from xpcom import components

from xpcom.server import UnwrapObject

import directoryServiceUtils

log = logging.getLogger("codeintel.udl")

# log.setLevel(logging.DEBUG)

# XXX We need to have a better mechanism for rationalizing and sharing

# common lexer style classes. For now we'll just HACKily grab from

# Komodo's styles.py. Some of this is duplicating logic in

# KoLanguageServiceBase.py.

_ko_src_dir = normpath(join(dirname(__file__), *([os.pardir]*3)))

sys.path.insert(0, join(_ko_src_dir, "schemes"))

try:

import styles

finally:

del sys.path[0]

del _ko_src_dir

#---- module interface

# Test 'udl/general/is_udl_x_style' tests these.

def is_udl_m_style(style):

return (ScintillaConstants.SCE_UDL_M_DEFAULT <= style

<= ScintillaConstants.SCE_UDL_M_COMMENT)

def is_udl_css_style(style):

return (ScintillaConstants.SCE_UDL_CSS_DEFAULT <= style

<= ScintillaConstants.SCE_UDL_CSS_OPERATOR)

def is_udl_csl_style(style):

return (ScintillaConstants.SCE_UDL_CSL_DEFAULT <= style

<= ScintillaConstants.SCE_UDL_CSL_REGEX)

def is_udl_ssl_style(style):

return (ScintillaConstants.SCE_UDL_SSL_DEFAULT <= style

<= ScintillaConstants.SCE_UDL_SSL_VARIABLE)

def is_udl_tpl_style(style):

return (ScintillaConstants.SCE_UDL_TPL_DEFAULT <= style

<= ScintillaConstants.SCE_UDL_TPL_VARIABLE)

# XXX Redundant code from koUDLLanguageBase.py::KoUDLLanguage

# Necessary because SilverCity.WordList splits input on white-space

_re_bad_filename_char = re.compile(r'([% \x80-\xff])')

def _lexudl_path_escape(m):

return '%%%02X' % ord(m.group(1))

def _urlescape(s):

return _re_bad_filename_char.sub(_lexudl_path_escape, s)

class UDLLexer(Lexer):

"""LexUDL wants the path to the .lexres file as the first element of

the first keywords list.

"""

_lock = threading.Lock()

_lexresfile_from_lang = None

_extra_lexer_dirs = set()

def __init__(self):

self._properties = SilverCity.PropertySet()

self._lexer = SilverCity.find_lexer_module_by_id(

ScintillaConstants.SCLEX_UDL)

lexres_path = _urlescape(self._get_lexres_path())

log.debug("escaped lexres_path: %r", lexres_path)

self._keyword_lists = [

SilverCity.WordList(lexres_path),

]

def tokenize_by_style(self, text, call_back=None):

"""LexUDL.cxx currently isn't thread-safe."""

self._lock.acquire()

try:

return Lexer.tokenize_by_style(self, text, call_back)

finally:

self._lock.release()

@staticmethod

def add_extra_lexer_dirs(dirs):

UDLLexer._extra_lexer_dirs.update(dirs)

UDLLexer._lexresfile_from_lang = None

if _xpcom_:

# Presume we are running under Komodo. Look in the available

# lexres dirs from extensions.

@staticmethod

def _generate_lexer_mapping():

"""Return dict {name > filename} of all lexer resource files (i.e.

those ones that can include compiled UDL .lexres files).

It yields directories that should "win" first.

"""

from glob import glob

lexresfile_from_lang = {}

koDirs = components.classes["@activestate.com/koDirs;1"] \

.getService(components.interfaces.koIDirs)

# Find all possible lexer dirs.

lexer_dirs = []

lexer_dirs.append(join(koDirs.userDataDir, "lexers")) # user

for extensionDir in directoryServiceUtils.getExtensionDirectories():

lexer_dirs.append(join(

extensionDir, "lexers")) # user-install extensions

lexer_dirs.append(join(

koDirs.commonDataDir, "lexers")) # site/common

lexer_dirs.append(join(koDirs.supportDir, "lexers")) # factory

for extra_dir in UDLLexer._extra_lexer_dirs:

lexer_dirs.append(extra_dir)

# Find all .lexeres files in these lexer dirs.

for d in reversed(lexer_dirs): # first come, first served

lexer_files = glob(join(d, "*.lexres"))

for f in lexer_files:

# Get lowered name without the ".lexres" extension.

name = basename(f).lower().rsplit(".", 1)[0]

lexresfile_from_lang[name] = f

return lexresfile_from_lang

else:

@staticmethod

def _generate_lexer_mapping():

"""Return dict {name > filename} of all lexer resource files (i.e.

those ones that can include compiled UDL .lexres files).

It yields directories that should "win" first.

"""

from glob import glob

lexresfile_from_lang = {}

# Find all possible lexer dirs.

lexer_dirs = []

lexer_dirs.append(join(dirname(__file__), "lexers"))

for extra_dir in UDLLexer._extra_lexer_dirs:

lexer_dirs.append(extra_dir)

# Find all .lexeres files in these lexer dirs.

for d in reversed(lexer_dirs): # first come, first served

lexer_files = glob(join(d, "*.lexres"))

for f in lexer_files:

# Get lowered name without the ".lexres" extension.

name = basename(f).lower().rsplit(".", 1)[0]

lexresfile_from_lang[name] = f

return lexresfile_from_lang

def _get_lexres_path(self):

lexresfile_from_lang = UDLLexer._lexresfile_from_lang

if lexresfile_from_lang is None:

# Generate and cache it.

lexresfile_from_lang = self._generate_lexer_mapping()

UDLLexer._lexresfile_from_lang = lexresfile_from_lang

lexres_file = lexresfile_from_lang.get(self.lang.lower())

if lexres_file is None:

raise CodeIntelError("could not find lexres file for %s: "

"`%s.lexres' does not exist in any "

"of the lexer dirs"

% (self.lang, self.lang))

return lexres_file

class UDLBuffer(CitadelBuffer):

"""A CodeIntel Buffer for a UDL-lexer-based language."""

sce_prefixes = ["SCE_UDL_"]

# XXX Not sure that this indirection will be useful, but we'll see.

# Sub-classes must set the following of these that are appropriate:

m_lang = None

css_lang = None

csl_lang = None

ssl_lang = None

tpl_lang = None

def lang_from_style(self, style):

if (ScintillaConstants.SCE_UDL_M_DEFAULT <= style

<= ScintillaConstants.SCE_UDL_M_COMMENT):

return self.m_lang

elif (ScintillaConstants.SCE_UDL_CSS_DEFAULT <= style

<= ScintillaConstants.SCE_UDL_CSS_OPERATOR):

return self.css_lang

elif (ScintillaConstants.SCE_UDL_CSL_DEFAULT <= style

<= ScintillaConstants.SCE_UDL_CSL_REGEX):

return self.csl_lang

elif (ScintillaConstants.SCE_UDL_SSL_DEFAULT <= style

<= ScintillaConstants.SCE_UDL_SSL_VARIABLE):

return self.ssl_lang

elif (ScintillaConstants.SCE_UDL_TPL_DEFAULT <= style

<= ScintillaConstants.SCE_UDL_TPL_VARIABLE):

return self.tpl_lang

else:

raise ValueError("unknown UDL style: %r" % style)

def lang_from_pos(self, pos):

style = self.accessor.style_at_pos(pos)

return self.lang_from_style(style)

_udl_family_from_lang_cache = None

@property

def udl_family_from_lang(self):

if self._udl_family_from_lang_cache is None:

self._udl_family_from_lang_cache = dict(

(uf, L) for (uf, L) in [

(self.m_lang, "M"),

(self.css_lang, "CSS"),

(self.csl_lang, "CSL"),

(self.ssl_lang, "SSL"),

(self.tpl_lang, "TPL"),

]

if L is not None

)

return self._udl_family_from_lang_cache

def text_chunks_from_lang(self, lang):

"""Generate a list of text chunks of the given language content.

For a single-language buffer this is trivial: 1 chunk of the whole

buffer. For multi-language buffers, less so.

Generates 2-tuples:

(POSITION-OFFSET, TEXT-CHUNK)

"""

udl_family_from_lang = self.udl_family_from_lang

if len(udl_family_from_lang) == 1:

yield 0, self.accessor.text

elif lang not in udl_family_from_lang:

pass

elif hasattr(self.accessor, "udl_family_chunk_ranges"):

udl_family = self.udl_family_from_lang[lang]

text = self.accessor.text # Note: assuming here that `text` is in *bytes*

for u, start, end in self.accessor.udl_family_chunk_ranges:

if u == udl_family:

yield start, text[start:end]

else:

min_style, max_style = {

self.m_lang: (ScintillaConstants.SCE_UDL_M_DEFAULT,

ScintillaConstants.SCE_UDL_M_COMMENT),

self.css_lang: (ScintillaConstants.SCE_UDL_CSS_DEFAULT,

ScintillaConstants.SCE_UDL_CSS_OPERATOR),

self.csl_lang: (ScintillaConstants.SCE_UDL_CSL_DEFAULT,

ScintillaConstants.SCE_UDL_CSL_REGEX),

self.ssl_lang: (ScintillaConstants.SCE_UDL_SSL_DEFAULT,

ScintillaConstants.SCE_UDL_SSL_VARIABLE),

self.tpl_lang: (ScintillaConstants.SCE_UDL_TPL_DEFAULT,

ScintillaConstants.SCE_UDL_TPL_VARIABLE),

}[lang]

in_chunk = False

pos_offset = None

text = self.accessor.text

for token in self.accessor.gen_tokens():

if in_chunk:

if not (min_style <= token["style"] <= max_style):

# SilverCity indeces are inclusive at the end.

end_index = token["end_index"] + 1

yield pos_offset, text[pos_offset:end_index]

in_chunk = False

else:

if min_style <= token["style"] <= max_style:

in_chunk = True

pos_offset = token["start_index"]

if in_chunk:

yield pos_offset, text[pos_offset:]

def scoperef_from_pos(self, pos):

"""Return the scoperef for the given position in this buffer.

A "scoperef" is a 2-tuple:

(<blob>, <lpath>)

where <blob> is the ciElementTree blob for the buffer content

and <lpath> is an ordered list of names into the blob

identifying the scope.

If no relevant scope is found (e.g. for example, in markup

content in PHP) then None is returned.

See Buffer.scoperef_from_pos() docstring for more details.

"""

lang = self.lang_from_pos(pos)

try:

blob = self.blob_from_lang[lang]

except KeyError:

return None

line = self.accessor.line_from_pos(pos) + 1 # convert to 1-based

return self.scoperef_from_blob_and_line(blob, line)

def trg_from_pos(self, pos, implicit=True):

if pos == 0:

return None

lang = self.lang_from_pos(pos-1)

if lang is None:

style = self.accessor.style_at_pos(pos)

style_names = self.style_names_from_style_num(style)

raise CodeIntelError("got unexpected style in `%s': %s %s"

% (basename(self.path), style, style_names))

try:

langintel = self.mgr.langintel_from_lang(lang)

except KeyError:

return None

return langintel.trg_from_pos(self, pos, implicit=implicit)

def preceding_trg_from_pos(self, pos, curr_pos):

if curr_pos == 0:

return None

lang = self.lang_from_pos(curr_pos-1)

try:

langintel = self.mgr.langintel_from_lang(lang)

except KeyError:

return None

return langintel.preceding_trg_from_pos(self, pos, curr_pos)

def curr_calltip_arg_range(self, trg_pos, calltip, curr_pos):

if curr_pos == 0:

return None

lang = self.lang_from_pos(curr_pos-1)

try:

langintel = self.mgr.langintel_from_lang(lang)

except KeyError:

return (-1, -1)

try:

return langintel.curr_calltip_arg_range(self, trg_pos, calltip,

curr_pos)

except AttributeError:

# This can happen if we accidentally move into a non-programming

# language during a calltip. E.g. bug 69529. Cancel the calltip

# in this case.

return (-1, -1)

def async_eval_at_trg(self, trg, ctlr):

try:

langintel = self.mgr.langintel_from_lang(trg.lang)

except KeyError:

return None

return langintel.async_eval_at_trg(self, trg, ctlr)

# Override Citadel.defn_trg_from_pos()

def defn_trg_from_pos(self, pos, lang=None):

# Work out the language from the position, as the citadel buffer will

# use the buffer language, we want a language specific to this pos.

return CitadelBuffer.defn_trg_from_pos(self, pos,

lang=self.lang_from_pos(pos))

def libs(self):

"""A simple `.libs' property does not work for multi-lang buffers.

Use `.libs_from_lang(lang)' instead.

"""

raise RuntimeError("`.libs' invalid for multi-lang buffers: use "

"`mgr.langintel_from_lang(lang).libs_from_buf(buf)' "

"directly")

def style_names_from_style_num(self, style_num):

# XXX Would like to have python-foo instead of p_foo or SCE_P_FOO, but

# that requires a more comprehensive solution for all langs and

# multi-langs.

style_names = []

# Get the constant name from ScintillaConstants.

if "UDL" not in self._style_name_from_style_num_from_lang:

name_from_num \

= self._style_name_from_style_num_from_lang["UDL"] = {}

if self.sce_prefixes is None:

raise CodeIntelError("'sce_prefixes' not set on class %s: cannot "

"determine style constant names"

% self.__class__.__name__)

for attr in dir(ScintillaConstants):

for sce_prefix in self.sce_prefixes:

if attr.startswith(sce_prefix):

name_from_num[getattr(ScintillaConstants, attr)] = attr

else:

name_from_num \

= self._style_name_from_style_num_from_lang["UDL"]

const_name = name_from_num[style_num]

style_names.append(const_name)

# Get a style group from styles.py.

if "UDL" in styles.StateMap:

for style_group, const_names in list(styles.StateMap["UDL"].items()):

if const_name in const_names:

style_names.append(style_group)

break

else:

log.warn("lang '%s' not in styles.StateMap: won't have "

"common style groups in HTML output" % "UDL")

return style_names

__string_styles = None

def string_styles(self):

if self.__string_styles is None:

state_map = styles.StateMap["UDL"]

self.__string_styles = [

getattr(ScintillaConstants, style_name)

for style_class in ("strings", "stringeol")

for style_name in state_map.get(style_class, [])

]

return self.__string_styles

__comment_styles = None

def comment_styles(self):

if self.__comment_styles is None:

state_map = styles.StateMap["UDL"]

self.__comment_styles = [

getattr(ScintillaConstants, style_name)

for style_class in ("comments", "here documents",

"data sections")

for style_name in state_map.get(style_class, [])

]

return self.__comment_styles

__number_styles = None

def number_styles(self):

if self.__number_styles is None:

state_map = styles.StateMap["UDL"]

self.__number_styles = [

getattr(ScintillaConstants, style_name)

for style_class in ("numbers",)

for style_name in state_map.get(style_class, [])

]

return self.__number_styles

class XMLParsingBufferMixin(object):

"""A mixin for UDLBuffer-based buffers of XML-y/HTML-y languages to

support the following:

- An "xml_tree" attribute that is a XML parse tree of the document

(lazily done from koXMLTreeService)

- An "xml_parse()" method to force a re-parse of the document.

TODO: locking?

"""

_xml_tree_cache = None

_xml_default_dataset_info = None

@property

def xml_tree(self):

if self._xml_tree_cache is None:

self.xml_parse()

return self._xml_tree_cache

def xml_parse(self):

from koXMLTreeService import getService

path = self.path

if isUnsavedPath(self.path):

# The "<Unsaved>/..." special path can *crash* Python if trying to

# open it. Besides, the "<Unsaved>" business is an internal

# codeintel detail.

path = None

self._xml_tree_cache = getService().getTreeForURI(

path, self.accessor.text)

def xml_default_dataset_info(self, node=None):

if self._xml_default_dataset_info is None:

import koXMLDatasetInfo

datasetSvc = koXMLDatasetInfo.getService()

self._xml_default_dataset_info = (

datasetSvc.getDefaultPublicId(self.m_lang, self.env),

None,

datasetSvc.getDefaultNamespace(self.m_lang, self.env))

return self._xml_default_dataset_info

def xml_tree_handler(self, node=None):

import koXMLDatasetInfo

return koXMLDatasetInfo.get_tree_handler(self._xml_tree_cache, node, self.xml_default_dataset_info(node))

def xml_node_at_pos(self, pos):

import koXMLTreeService

self.xml_parse()

tree = self._xml_tree_cache

if not tree:

return None

line, col = self.accessor.line_and_col_at_pos(pos)

node = tree.locateNode(line, col)

# XXX this needs to be worked out better

last_start = self.accessor.text.rfind('<', 0, pos)

last_end = self.accessor.text.find('>', last_start, pos)

if node is None and last_start >= 0:

node = koXMLTreeService.elementFromText(

tree, self.accessor.text[last_start:last_end], node)

if node is None or node.start is None:

return node

# elementtree line numbers are 1 based, convert to zero based

node_pos = self.accessor.pos_from_line_and_col(

node.start[0]-1, node.start[1])

if last_end == -1 and last_start != node_pos:

# print "try parse ls %d le %d np %d pos %d %r" % (last_start, last_end, node_pos, pos, accessor.text[last_start:pos])

# we have a dirty tree, need to create a current node and add it

newnode = koXMLTreeService.elementFromText(

tree, self.accessor.text[last_start:pos], node)

if newnode is not None:

return newnode

return node

class _NotYetSet(object):

# Used below to distinguish from None.

pass

class UDLCILEDriver(CILEDriver):

ssl_lang = None # Sub-classes must set one or both of these for

csl_lang = None # citadel-scanning support.

_master_cile_driver = None

slave_csl_cile_driver = _NotYetSet # to distinguish from None

@property

def master_cile_driver(self):

"""The primary CILE driver for this multi-lang lang.

This is the CILE driver for the SSL lang, if there is one, otherwise

for the csl_lang.

Side effect: `self.slave_csl_cile_driver' is determined the

first time this is called. A little gross, I know, but it

avoids having a separate property.

"""

if self._master_cile_driver is None:

if self.ssl_lang is not None:

self._master_cile_driver \

= self.mgr.citadel.cile_driver_from_lang(self.ssl_lang)

self.slave_csl_cile_driver \

= self.mgr.citadel.cile_driver_from_lang(self.csl_lang)

else:

self._master_cile_driver \

= self.mgr.citadel.cile_driver_from_lang(self.csl_lang)

self.slave_csl_cile_driver = None

return self._master_cile_driver

def scan_purelang(self, buf):

log.info("scan_purelang: path: %r lang: %s", buf.path, buf.lang)

return self.master_cile_driver.scan_multilang(

buf, self.slave_csl_cile_driver)