tests pass on python2 and python3

youtube · mattip · Oct 5, 2019 · Jan 7, 2020 · Jan 7, 2020 · Jan 7, 2020
commit 93992513888a900aa36649aca13efa3fccf32cb6
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,10 @@
 /build
 /spitfire.egg-info
 /spitfire/compiler/parser.py
+/tests/*.py
 
 # Local pyenv version files
 .python-version
+
+# Artifacts
+*.swp
diff --git a/Makefile b/Makefile
@@ -27,7 +27,7 @@ else
 endif
 
 COMPILER = PYTHONPATH=. $(PYTHON) scripts/spitfire-compile
-CRUNNER = $(PYTHON) scripts/crunner.py
+CRUNNER = PYTHONPATH=. $(PYTHON) scripts/crunner.py
 UNITTEST = $(PYTHON) -m unittest
 YAPPS = $(PYTHON) third_party/yapps2/yapps2.py
 

diff --git a/scripts/crunner.py b/scripts/crunner.py
@@ -130,8 +130,11 @@ def process_file(self, filename):
 
         classname = util.filename2classname(filename)
         modulename = util.filename2modulename(filename)
+        suffix = '.%d' % sys.version_info[0]
         test_output_path = os.path.join(self.options.test_output,
                                         classname + '.txt')
+        if os.path.exists(test_output_path + suffix):
+            test_output_path += suffix
 
         if self.options.verbose:
             sys.stderr.write(modulename + ' ... ')
@@ -165,7 +168,7 @@ def process_file(self, filename):
                 if 'source_code' in self.options.debug_flags:
                     print("source_code:", file=buffer)
                     for i, line in enumerate(self.compiler._source_code.split(
-                            '\n')):
+                            b'\n')):
                         print('% 3s' % (i + 1), line, file=buffer)
 
         test_failed = False
@@ -191,7 +194,10 @@ def process_file(self, filename):
                 try:
                     template_class = getattr(template_module, classname)
                     template = template_class(search_list=self.search_list)
-                    current_output = template.main().encode('utf8')
+                    if sys.version_info[0] < 3:
+                        current_output = template.main().encode('utf8')
+                    else:
+                        current_output = template.main()
                 except Exception as e:
                     # An exception here doesn't meant that the test fails
                     # necessarily since libraries don't have a class; as long as
@@ -218,6 +224,7 @@ def process_file(self, filename):
                 test_output = None
             else:
                 test_output = test_file.read()
+                test_file.close()
                 if current_output != test_output:
                     test_failed = True
                     if self.options.debug:
@@ -264,8 +271,9 @@ def process_file(self, filename):
 
 
 if __name__ == '__main__':
-    reload(sys)
-    sys.setdefaultencoding('utf8')
+    if sys.version_info[0] < 3:
+        reload(sys)
+        sys.setdefaultencoding('utf8')
 
     option_parser = optparse.OptionParser()
     options.add_common_options(option_parser)

diff --git a/scripts/spitfire-compile b/scripts/spitfire-compile
@@ -48,8 +48,9 @@ def process_file(spt_compiler, filename, options):
 
 
 if __name__ == '__main__':
-    reload(sys)
-    sys.setdefaultencoding('utf8')
+    if sys.version_info[0] < 3:
+        reload(sys)
+        sys.setdefaultencoding('utf8')
 
     option_parser = optparse.OptionParser()
     options.add_common_options(option_parser)

diff --git a/spitfire/compiler/codegen.py b/spitfire/compiler/codegen.py
@@ -4,8 +4,11 @@
 # license that can be found in the LICENSE file.
 
 import logging
-
-import cStringIO as StringIO
+import sys
+if sys.version_info[0] < 3:
+    import cStringIO as io
+else:
+    import io
 
 from spitfire.compiler import ast
 
@@ -58,7 +61,7 @@ def __init__(self, ast_root, compiler, options=None):
         # the stack.
         self.function_stack = []
         self.options = options
-        self.output = StringIO.StringIO()
+        self.output = io.StringIO()
         self.template = None
         self.baked_mode = False
 
@@ -70,7 +73,7 @@ def get_code(self):
     def generate_python(self, code_node):
         try:
             return code_node.src_line
-        except AttributeError, e:
+        except AttributeError as e:
             self.compiler.error(CodegenError("can't write code_node: %s\n\t%s" %
                                              (code_node, e)))
 
@@ -694,7 +697,7 @@ def codegenDefault(self, node):
         try:
             return [CodeNode(line % vars(node))
                     for line in v['AST%s_tmpl' % node.__class__.__name__]]
-        except KeyError, e:
+        except KeyError as e:
             self.compiler.error(CodegenError("no codegen for %s %s" % (type(
                 node), vars(node))))
 

diff --git a/spitfire/compiler/compiler.py b/spitfire/compiler/compiler.py
@@ -296,7 +296,7 @@ def write_src_file(self, src_code):
 
         outfile_path = os.path.join(self.output_directory, relative_dir,
                                     outfile_name)
-        outfile = open(outfile_path, 'w')
+        outfile = open(outfile_path, 'wb')
         outfile.write(src_code)
         outfile.close()
 

diff --git a/spitfire/compiler/macros/i18n.py b/spitfire/compiler/macros/i18n.py
@@ -5,7 +5,10 @@
 
 import sys
 
-import cStringIO as StringIO
+if sys.version_info[0] < 3:
+    import StringIO as StringIO
+else:
+    import io as StringIO
 
 from spitfire.compiler import analyzer
 from spitfire.compiler import ast
@@ -57,7 +60,7 @@ def macro_i18n(macro_node, arg_map, compiler):
     # most apps will have to stub this part out somehow i think
     macro_content_ast = util.parse(macro_node.value, 'i18n_goal')
     i18n_msg = make_i18n_message(macro_node.value, macro_content_ast)
-    i18n_msg_utf8 = i18n_msg.encode(sys.getdefaultencoding())
+    i18n_msg_utf8 = i18n_msg.encode('utf-8')
     #print "macro_content_ast"
     #print "orginal:", macro_node.value
     #print "i18n:", i18n_msg_utf8
@@ -76,5 +79,5 @@ def macro_function_i18n(call_node, arg_map, compiler):
         raise analyzer.SemanticAnalyzerError(
             '$i18n argument "%s" must be a string literal' % msg_arg_node)
     i18n_msg = text.i18n_mangled_message(msg_arg_node.value)
-    i18n_msg_utf8 = i18n_msg.encode(sys.getdefaultencoding())
+    i18n_msg_utf8 = i18n_msg.encode('utf-8')
     return u"'%s'" % i18n_msg.replace("'", "\\'")
diff --git a/spitfire/compiler/util.py b/spitfire/compiler/util.py
@@ -4,10 +4,11 @@
 # license that can be found in the LICENSE file.
 
 import logging
-import new
+from types import ModuleType
 import os.path
 import re
 import sys
+import io
 
 from spitfire.compiler import options
 from spitfire.compiler import parser
@@ -54,13 +55,20 @@ def parse_template(src_text, xspt_mode=False):
         return parse(src_text)
 
 
-def read_template_file(filename):
-    f = open(filename, 'r')
-    try:
-        return f.read().decode('utf8')
-    finally:
-        f.close()
-
+if sys.version_info[0] < 3:
+    def read_template_file(filename):
+        f = open(filename, 'r')
+        try:
+            return f.read().decode('utf-8')
+        finally:
+            f.close()
+else:
+    def read_template_file(filename):
+        f = open(filename, 'r', encoding='utf-8')
+        try:
+            return f.read()
+        finally:
+            f.close()
 
 def read_function_registry(filename):
     f = open(filename)
@@ -137,11 +145,11 @@ def load_template(template_src,
 
 # a helper method to import a template without having to save it to disk
 def load_module_from_src(src_code, filename, module_name):
-    module = new.module(module_name)
+    module = ModuleType(module_name)
     sys.modules[module_name] = module
 
     bytecode = compile(src_code, filename, 'exec')
-    exec bytecode in module.__dict__
+    exec(bytecode, module.__dict__)
     return module
 
 

diff --git a/spitfire/compiler/xhtml2ast.py b/spitfire/compiler/xhtml2ast.py
@@ -4,6 +4,7 @@
 # license that can be found in the LICENSE file.
 
 import sys
+import io
 import traceback
 import xml.dom.minidom
 
@@ -27,7 +28,7 @@ class XHTML2AST(object):
     attr_op_namespace = 'pyattr'
 
     def build_template(self, filename):
-        f = open(filename)
+        f = io.open(filename, 'r', encoding='utf-8')
         data = f.read().decode('utf8')
         f.close()
         return self.parse(data)

diff --git a/spitfire/runtime/filters.py b/spitfire/runtime/filters.py
@@ -9,6 +9,10 @@
 import types
 from spitfire import runtime
 from spitfire.runtime import udn
+import sys
+if sys.version_info[0] > 2:
+    unicode = basestring = str
+    long = int
 
 
 # decorate a function object so the default filter will not be applied to the

diff --git a/spitfire/runtime/runner.py b/spitfire/runtime/runner.py
@@ -8,7 +8,10 @@
 import os.path
 import sys
 
-import cPickle as pickle
+if sys.version_info[0] < 3:
+    import cPickle as pickle
+else:
+    import pickle
 
 
 def run_template(class_object):

diff --git a/spitfire/runtime/template.py b/spitfire/runtime/template.py
@@ -5,8 +5,6 @@
 
 # an 'abstract' base class for a template, seems like a good idea for now
 
-import cStringIO as StringIO
-
 from spitfire import runtime
 from spitfire.runtime import baked
 from spitfire.runtime import filters

diff --git a/spitfire/test_util.py b/spitfire/test_util.py
@@ -3,7 +3,7 @@
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
 
-from builtins import object
+#from builtins import object
 import sys
 
 

diff --git a/tests/output-preserve-whitespace/some_library.txt.3 b/tests/output-preserve-whitespace/some_library.txt.3
@@ -0,0 +1 @@
+module 'tests.some_library' has no attribute 'some_library'
diff --git a/tests/output-preserve-whitespace/template_if_4.txt.3 b/tests/output-preserve-whitespace/template_if_4.txt.3
@@ -0,0 +1,2 @@
+    test ' '
+
diff --git a/tests/output-preserve-whitespace/util.txt.3 b/tests/output-preserve-whitespace/util.txt.3
@@ -0,0 +1 @@
+module 'tests.util' has no attribute 'util'
diff --git a/tests/output/some_library.txt.3 b/tests/output/some_library.txt.3
@@ -0,0 +1 @@
+module 'tests.some_library' has no attribute 'some_library'
diff --git a/tests/output/template_if_4.txt.3 b/tests/output/template_if_4.txt.3
@@ -0,0 +1 @@
+  test ' '
diff --git a/tests/output/util.txt.3 b/tests/output/util.txt.3
@@ -0,0 +1 @@
+module 'tests.util' has no attribute 'util'
diff --git a/tests/test-hoist-conditional-loop-optimization-3.txt b/tests/test-hoist-conditional-loop-optimization-3.txt
@@ -3,7 +3,7 @@
 ## the bug was in calculating the dependencies for $msg
 ## you have to take into account the dependency of the
 ## and conditions between you and the loop
-#for $i in $xrange(2)
+#for $i in $range(2)
   #if $i % 2 == 0
     #set $msg = 'Success: even'
   #else

diff --git a/third_party/yapps2/yappsrt.py b/third_party/yapps2/yappsrt.py
@@ -1,12 +1,15 @@
 # Yapps 2.0 Runtime
 #
 # This module is needed to run generated parsers.
+from __future__ import division, print_function
 
-from string import join, count, find, rfind
 import logging
 import re
-import StringIO
 import sys
+if sys.version_info[0] < 3:
+    import StringIO
+else:
+    import io as StringIO
 
 class SyntaxError(Exception):
   """When we run into an unexpected token, this is the exception to use"""
@@ -109,7 +112,7 @@ def scan(self, restrict):
       if best_pat == '(error)' and best_match < 0:
         msg = "Bad Token"
         if restrict:
-          msg = "Trying to find one of "+join(restrict,", ")
+          msg = "Trying to find one of "+ ', '.join(restrict)
         raise SyntaxError(self.pos, msg)
 
       # If we found something that isn't to be ignored, return it
@@ -157,25 +160,25 @@ def format_error(input, err, scanner):
   """This is a really dumb long function to print error messages nicely."""
   error_message = StringIO.StringIO()
   p = err.pos
-  print >> error_message, "error position", p
+  print("error position", p, file=error_message)
   # Figure out the line number
-  line = count(input[:p], '\n')
-  print >> error_message, err.msg, "on line", repr(line+1) + ":"
+  line = input[:p].count('\n')
+  print(err.msg, "on line", repr(line+1) + ":", file=error_message)
   # Now try printing part of the line
   text = input[max(p-80, 0):p+80]
   p = p - max(p-80, 0)
 
   # Strip to the left
-  i = rfind(text[:p], '\n')
-  j = rfind(text[:p], '\r')
+  i = text[:p].rfind('\n')
+  j = text[:p].rfind('\r')
   if i < 0 or (0 <= j < i): i = j
   if 0 <= i < p:
     p = p - i - 1
   text = text[i+1:]
 
   # Strip to the right
-  i = find(text,'\n', p)
-  j = find(text,'\r', p)
+  i = text.find('\n', p)
+  j = text.find('\r', p)
   if i < 0 or (0 <= j < i):
     i = j
   if i >= 0:
@@ -188,22 +191,22 @@ def format_error(input, err, scanner):
     p = p - 7
 
   # Now print the string, along with an indicator
-  print >> error_message, '> ', text.replace('\t', ' ').encode(sys.getdefaultencoding())
-  print >> error_message, '> ', ' '*p + '^'
-  print >> error_message, 'List of nearby tokens:', scanner
+  print('> ', text.replace('\t', ' ').encode(sys.getdefaultencoding()), file=error_message)
+  print('> ', ' '*p + '^', file=error_message)
+  print('List of nearby tokens:', scanner, file=error_message)
   return error_message.getvalue()
 
 
 def wrap_error_reporter(parser, rule):
   try:
     return getattr(parser, rule)()
-  except SyntaxError, e:
+  except SyntaxError as e:
     logging.exception('syntax error')
     input = parser._scanner.input
     try:
       error_msg = format_error(input, e, parser._scanner)
     except ImportError:
-      error_msg = 'Syntax Error %s on line\n' % (e.msg, 1 + count(input[:e.pos]))
-  except NoMoreTokens, e:
+      error_msg = 'Syntax Error %s on line\n' % (e.msg, 1 + input[:e.pos].count())
+  except NoMoreTokens as e:
     error_msg = 'Could not complete parsing; stopped around here:\n%s\n%s' % (parser._scanner, e)
   raise FatalParseError(error_msg)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		module 'tests.some_library' has no attribute 'some_library'