fixed some unicode hiccups

samizdatco · samizdatco · commit 276fb6a62ebf · 2014-06-26T14:19:36.000-04:00
- compile() is a little finnicky. everything’s great if you pass it a
unicode source string *unless* the string contains a line like `#
encoding: foo` (which raises a syntax error)
- to get around this, we encode the script back to its preferred
encoding then pass the bytestring to compile()
- as a side-effect, the document class is now careful to honor the
encoding string while reading in the text of a .pv file (but seriously,
just use utf-8. it’s on by default…)
diff --git a/plotdevice/gui/document.py b/plotdevice/gui/document.py
@@ -10,20 +10,19 @@
 from Foundation import *
 from AppKit import *
 
-# from plotdevice.gui.preferences import editor_info
-from plotdevice.gui.editor import OutputTextView, EditorView
-from plotdevice.gui.widgets import DashboardController, ExportSheet
-from plotdevice.gui.views import FullscreenWindow, FullscreenView
-from plotdevice.gui import set_timeout
-from plotdevice.run import Sandbox
-from plotdevice import util
+from .editor import OutputTextView, EditorView
+from .widgets import DashboardController, ExportSheet
+from .views import FullscreenWindow, FullscreenView
+from ..run import Sandbox, encoding
+from . import set_timeout
 
 NSEventGestureAxisVertical = 2
 
 class PlotDeviceDocument(NSDocument):
     def init(self):
         self.stationery = "TMPL:sketch" # untitled/example docs flag
         self.source = None # string read in from file
+        self.source_enc = 'utf-8' # or the contents of a coding: comment
         self.script = None # window controller
         return super(PlotDeviceDocument,self).init()
 
@@ -54,14 +53,18 @@ def setFileURL_(self, url):
 
     def writeToURL_ofType_error_(self, url, tp, err):
         path = url.fileSystemRepresentation()
-        text = self.script._get_source().encode("utf8")
+        src = self.script._get_source()
+        # always use the same encoding we were read in with
+        text = src.encode(self.source_enc)
         with file(path, 'w', 0) as f:
             f.write(text)
         return True, err
 
     def readFromURL_ofType_error_(self, url, tp, err):
         path = url.fileSystemRepresentation()
-        self.source = file(path).read().decode("utf-8")
+        src = file(path).read()
+        self.source_enc = encoding(src) or 'utf-8'
+        self.source = src.decode(self.source_enc)
         if self.script:
             self.script.setPath_source_(self.path, self.source)
         return True, err
diff --git a/plotdevice/run/__init__.py b/plotdevice/run/__init__.py
@@ -1,8 +1,20 @@
-import linecache
+import linecache, re
 from sys import exc_info
 from os.path import abspath, dirname, relpath
 from traceback import format_list, format_exception_only
 
+def encoding(src):
+    """Searches the first two lines of a string looking for an `# encoding: ???` comment."""
+    re_enc = re.compile(r'coding[=:]\s*([-\w.]+)')
+    for line in src.split('\n')[:2]:
+        if not line.strip().startswith('#'):
+            continue
+        m = re_enc.search(line)
+        if not m:
+            continue
+        return m.group(1)
+    return None
+
 def stacktrace(script=None, src=None):
     """print a clean traceback and optionally rewrite the paths relative to a script path"""
 
diff --git a/plotdevice/run/sandbox.py b/plotdevice/run/sandbox.py
@@ -6,7 +6,7 @@
 from PyObjCTools import AppHelper
 from Foundation import *
 from AppKit import *
-from ..run import stacktrace, coredump
+from ..run import stacktrace, coredump, encoding
 from ..lib.io import MovieExportSession, ImageExportSession
 from plotdevice import util, context, gfx, DeviceError
 
@@ -138,12 +138,17 @@ def compile(self, src=None):
         # self.__doc__ = {}
         # self.namespace.update(dict( __doc__=self.__doc__, ))
 
+
         result = Outcome(True, [])
         if not self._code:
             # Compile the script
             def compileScript():
                 scriptname = self._path or "<Untitled>"
-                self._code = compile("%s\n\n"%self._source, scriptname.encode('ascii', 'ignore'), "exec")
+                # src needs to be a bytestring if the script defines its encoding in an
+                # `encoding: ...` comment. otherwise just pass the unicode to compile()
+                enc = encoding(self._source)
+                src = self._source.encode(enc) if enc else self._source
+                self._code = compile(src, scriptname.encode('ascii', 'ignore'), "exec")
             result = self.call(compileScript)
             if not result.ok:
                 return result
