Discover and run unit tests from check.py (WebAssembly#1948)

tlively · web-flow · commit fe0b16aa2223 · 2019-03-19T18:18:20.000-07:00
unittest is Python's standard testing framework, so this change allows
arbitrary tests to be written without introducing any new dependencies
or code in check.py. A new test that was not possible to write before
is also included. It is the first of many.
diff --git a/check.py b/check.py
@@ -18,17 +18,22 @@
 import shutil
 import subprocess
 import sys
+import unittest
 
 from scripts.test.support import run_command, split_wast, node_test_glue, node_has_webassembly
 from scripts.test.shared import (
     BIN_DIR, MOZJS, NATIVECC, NATIVEXX, NODEJS, BINARYEN_JS,
     WASM_AS, WASM_CTOR_EVAL, WASM_OPT, WASM_SHELL, WASM_MERGE, WASM_METADCE,
     WASM_DIS, WASM_REDUCE, binary_format_check, delete_from_orbit, fail, fail_with_error,
     fail_if_not_identical, fail_if_not_contained, has_vanilla_emcc,
-    has_vanilla_llvm, minify_check, num_failures, options, tests,
-    requested, warnings, has_shell_timeout, fail_if_not_identical_to_file
+    has_vanilla_llvm, minify_check, options, tests, requested, warnings,
+    has_shell_timeout, fail_if_not_identical_to_file
 )
 
+# For shared.num_failures. Cannot import directly because modifications made in
+# shared.py would not affect the version imported here.
+import scripts.test.shared as shared
+
 import scripts.test.asm2wasm as asm2wasm
 import scripts.test.lld as lld
 import scripts.test.wasm2js as wasm2js
@@ -592,6 +597,17 @@ def run_gcc_tests():
       fail_if_not_identical_to_file(actual, expected)
 
 
+def run_unittest():
+  print '\n[ checking unit tests...]\n'
+
+  # equivalent to `python -m unittest discover -s ./test -v`
+  suite = unittest.defaultTestLoader.discover(os.path.dirname(options.binaryen_test))
+  result = unittest.TextTestRunner(verbosity=2, failfast=options.abort_on_first_failure).run(suite)
+  shared.num_failures += len(result.errors) + len(result.failures)
+  if options.abort_on_first_failure and shared.num_failures:
+    raise Exception("unittest failed")
+
+
 # Run all the tests
 def main():
   run_help_tests()
@@ -618,17 +634,20 @@ def main():
   if options.run_gcc_tests:
     run_gcc_tests()
 
+  run_unittest()
+
   # Check/display the results
-  if num_failures == 0:
+  if shared.num_failures == 0:
     print '\n[ success! ]'
 
   if warnings:
     print '\n' + '\n'.join(warnings)
 
-  if num_failures > 0:
-    print '\n[ ' + str(num_failures) + ' failures! ]'
+  if shared.num_failures > 0:
+    print '\n[ ' + str(shared.num_failures) + ' failures! ]'
+    return 1
 
-  return num_failures
+  return 0
 
 
 if __name__ == '__main__':
diff --git a/scripts/test/shared.py b/scripts/test/shared.py
@@ -281,6 +281,60 @@ def remove_readonly_and_try_again(func, path, exc_info):
     pass
 
 
+# This is a workaround for https://bugs.python.org/issue9400
+class Py2CalledProcessError(subprocess.CalledProcessError):
+  def __init__(self, returncode, cmd, output=None, stderr=None):
+    super(Exception, self).__init__(returncode, cmd, output, stderr)
+    self.returncode = returncode
+    self.cmd = cmd
+    self.output = output
+    self.stderr = stderr
+
+
+# https://docs.python.org/3/library/subprocess.html#subprocess.CompletedProcess
+class Py2CompletedProcess:
+  def __init__(self, args, returncode, stdout, stderr):
+    self.args = args
+    self.returncode = returncode
+    self.stdout = stdout
+    self.stderr = stderr
+
+  def __repr__(self):
+    _repr = ['args=%s, returncode=%s' % (self.args, self.returncode)]
+    if self.stdout is not None:
+      _repr += 'stdout=' + repr(self.stdout)
+    if self.stderr is not None:
+      _repr += 'stderr=' + repr(self.stderr)
+    return 'CompletedProcess(%s)' % ', '.join(_repr)
+
+  def check_returncode(self):
+    if self.returncode != 0:
+      raise Py2CalledProcessError(returncode=self.returncode, cmd=self.args,
+                                  output=self.stdout, stderr=self.stderr)
+
+
+def run_process(cmd, check=True, input=None, universal_newlines=True,
+                capture_output=False, *args, **kw):
+  kw.setdefault('universal_newlines', True)
+
+  if hasattr(subprocess, "run"):
+    ret = subprocess.run(cmd, check=check, input=input, *args, **kw)
+    return ret
+
+  # Python 2 compatibility: Introduce Python 3 subprocess.run-like behavior
+  if input is not None:
+    kw['stdin'] = subprocess.PIPE
+  if capture_output:
+    kw['stdout'] = subprocess.PIPE
+    kw['stderr'] = subprocess.PIPE
+  proc = subprocess.Popen(cmd, *args, **kw)
+  stdout, stderr = proc.communicate(input)
+  result = Py2CompletedProcess(cmd, proc.returncode, stdout, stderr)
+  if check:
+    result.check_returncode()
+  return result
+
+
 def fail_with_error(msg):
   global num_failures
   try:
diff --git a/test/crash/__init__.py b/test/crash/__init__.py
diff --git a/test/crash/test_features.py b/test/crash/test_features.py
@@ -0,0 +1,18 @@
+import unittest
+from scripts.test.shared import WASM_OPT, run_process
+
+
+class FeatureValidationTest(unittest.TestCase):
+  def test_simd_type(self):
+    module = """
+    (module
+     (func $foo (param $0 v128) (result v128)
+      (local.get $0)
+     )
+    )
+    """
+    p = run_process(WASM_OPT + ['--mvp-features', '--print'],
+                    input=module, check=False, capture_output=True)
+    self.assertIn("all used types should be allowed", p.stderr)
+    self.assertIn("Fatal: error in validating input", p.stderr)
+    self.assertNotEqual(p.returncode, 0)
