Enhance unit tests (cztomczak#59).

cztomczak · cztomczak · commit 15fb9f8eaf61 · 2016-09-19T08:47:01.000+02:00
Display summary with real number of tests ran including sub-tests in
main_test.

Ensure that messages are coming from Javascript in OnConsoleMessage.

Automate checking for True/False asserts in ClientHandler and External.
diff --git a/unittests/_test_runner.py b/unittests/_test_runner.py
@@ -3,7 +3,7 @@
 """Run unit tests. With no arguments all tests are run. Read notes below.
 
 Usage:
-    _runner.py [FILE | _TESTCASE]
+    _test_runner.py [FILE | _TESTCASE]
 
 Options:
     FILE       Run tests from single file
@@ -83,7 +83,7 @@ def run_testcase(self, testcase):
         self._discover("[!_]*.py", testcase)
         assert not self._count_suites(self._isolated_suites)
         if not self._count_suites(self._suites):
-            print("[_runner.py] ERROR: test case not found")
+            print("[_test_runner.py] ERROR: test case not found")
             sys.exit(1)
         # Import errors found during discovery are ignored
         self._run_suites(self._suites)
@@ -140,14 +140,17 @@ def _run_suites_in_isolation(self, suites):
             # Run test using new instance of Python interpreter
             try:
                 output = subprocess.check_output(
-                        ["python", "_runner.py", testcase_id],
+                        ["python", "_test_runner.py", testcase_id],
                         stderr=subprocess.STDOUT)
                 exit_code = 0
             except subprocess.CalledProcessError as exc:
                 output = exc.output
                 exit_code = exc.returncode
             if type(output) != str:
                 output = output.decode("utf-8", errors="replace")
+            match = re.search(r"Ran (\d+) sub-tests in \w+", output)
+            if match:
+                self.ran += int(match.group(1))
             sys.stdout.write(output)
             # If tests failed parse output for errors/failures
             if exit_code:
@@ -228,9 +231,9 @@ def _print_summary(self):
         # type: () -> None
         """Print summary and exit."""
         print("-"*70)
-        print("[_runner.py] Ran "+str(self.ran)+" tests in total")
+        print("[_test_runner.py] Ran "+str(self.ran)+" tests in total")
         if self.errors or self.failures:
-            failed_str = "[_runner.py] FAILED ("
+            failed_str = "[_test_runner.py] FAILED ("
             if self.failures:
                 failed_str += ("failures="+str(self.failures))
             if self.errors:
@@ -240,7 +243,7 @@ def _print_summary(self):
             failed_str += ")"
             print(failed_str)
         else:
-            print("[_runner.py] OK")
+            print("[_test_runner.py] OK")
         self._exit()
 
     def _exit(self):
diff --git a/unittests/isolated_test.py b/unittests/isolated_test.py
@@ -6,7 +6,7 @@
 
 import unittest
 # noinspection PyUnresolvedReferences
-import _runner
+import _test_runner
 from os.path import basename
 
 # Globals
@@ -35,4 +35,4 @@ def test_isolated3(self):
 
 
 if __name__ == "__main__":
-    _runner.main(basename(__file__))
+    _test_runner.main(basename(__file__))
diff --git a/unittests/main_test.py b/unittests/main_test.py
@@ -1,15 +1,16 @@
 """General testing of CEF Python."""
 
-# To show the window for an extended period of time increase this number.
-MESSAGE_LOOP_RANGE = 25  # each iteration is 0.01 sec
-
 import unittest
 # noinspection PyUnresolvedReferences
-import _runner
+import _test_runner
 from os.path import basename
 from cefpython3 import cefpython as cef
 import time
 import base64
+import sys
+
+# To show the window for an extended period of time increase this number.
+MESSAGE_LOOP_RANGE = 25  # each iteration is 0.01 sec
 
 g_browser = None
 g_client_handler = None
@@ -27,36 +28,41 @@
     </style>
     <script>
     function print(msg) {
+        console.log(msg+" [JS]");
         msg = msg.replace("ok", "<b style='color:green'>ok</b>");
         msg = msg.replace("error", "<b style='color:red'>error</b>");
         document.getElementById("console").innerHTML += msg+"<br>";
     }
     window.onload = function(){
         print("window.onload() ok");
 
-        print("test_property1 = <i>"+test_property1+"</i>")
+        // Test binding property: test_property1
         if (test_property1 == "Test binding property to the 'window' object") {
-            print("ok");
+            print("test_property_1 ok");
         } else {
-            print("error");
-            throw "test_property1 contains invalid string";
+            throw new Error("test_property1 contains invalid string");
         }
 
-        print("test_property2 = <i>"+JSON.stringify(test_property2)+"</i>");
+        // Test binding property: test_property2
         if (JSON.stringify(test_property2) == '{"key1":"Test binding property'+
                 ' to the \\'window\\' object","key2":["Inside list",1,2]}') {
-            print("ok");
+            print("test_property2 ok");
         } else {
-            print("error");
-            throw "test_property2 contains invalid value";
+            throw new Error("test_property2 contains invalid value");
         }
 
+        // Test binding function: test_function
         test_function();
         print("test_function() ok");
 
+        // Test binding external object and use of javascript<>python callbacks
         external.test_callbacks(function(msg_from_python, py_callback){
-            print("test_callbacks(): "+msg_from_python+" ok")
-            print("py_callback.toString()="+py_callback.toString());
+            if (msg_from_python == "String sent from Python") {
+                print("test_callbacks() ok");
+            } else {
+                throw new Error("test_callbacks(): msg_from_python contains"+
+                                " invalid value");
+            }
             py_callback("String sent from Javascript");
             print("py_callback() ok");
         });
@@ -73,34 +79,48 @@
 g_datauri = "data:text/html;base64,"+base64.b64encode(g_datauri_data.encode(
         "utf-8", "replace")).decode("utf-8", "replace")
 
+g_subtests_ran = 0
+
+
+def subtest_message(message):
+    global g_subtests_ran
+    g_subtests_ran += 1
+    print(str(g_subtests_ran) + ". " + message)
+    sys.stdout.flush()
+
 
 class MainTest_IsolatedTest(unittest.TestCase):
 
     def test_main(self):
         """Main entry point."""
         # All this code must run inside one single test, otherwise strange
         # things happen.
+        print("")
 
         # Test initialization of CEF
         cef.Initialize({
             "debug": False,
             "log_severity": cef.LOGSEVERITY_ERROR,
             "log_file": "",
         })
+        subtest_message("cef.Initialize() ok")
 
         # Test global client callback
         global g_client_handler
         g_client_handler = ClientHandler(self)
         cef.SetGlobalClientCallback("OnAfterCreated",
                                     g_client_handler._OnAfterCreated)
+        subtest_message("cef.SetGlobalClientCallback() ok")
 
         # Test creation of browser
         global g_browser
         g_browser = cef.CreateBrowserSync(url=g_datauri)
         self.assertIsNotNone(g_browser, "Browser object")
+        subtest_message("cef.CreateBrowserSync() ok")
 
         # Test client handler
         g_browser.SetClientHandler(g_client_handler)
+        subtest_message("browser.SetClientHandler() ok")
 
         # Test javascript bindings
         global g_external
@@ -112,70 +132,76 @@ def test_main(self):
         bindings.SetProperty("test_property2", g_external.test_property2)
         bindings.SetObject("external", g_external)
         g_browser.SetJavascriptBindings(bindings)
+        subtest_message("browser.SetJavascriptBindings() ok")
 
         # Run message loop for 0.5 sec.
         # noinspection PyTypeChecker
         for i in range(MESSAGE_LOOP_RANGE):
             cef.MessageLoopWork()
             time.sleep(0.01)
+        subtest_message("cef.MessageLoopWork() ok")
 
         # Test browser closing. Remember to clean reference.
         g_browser.CloseBrowser(True)
         g_browser = None
+        subtest_message("browser.CloseBrowser() ok")
 
         # Give it some time to close before calling shutdown.
         # noinspection PyTypeChecker
         for i in range(25):
             cef.MessageLoopWork()
             time.sleep(0.01)
 
-        # Client handler asserts
-        self.assertTrue(g_client_handler.OnAfterCreated_called,
-                        "OnAfterCreated() call")
-        self.assertTrue(g_client_handler.OnLoadStart_called,
-                        "OnLoadStart() call")
-        self.assertTrue(g_client_handler.OnLoadEnd_called,
-                        "OnLoadEnd() call")
-        self.assertTrue(g_client_handler.FrameSourceVisitor_called,
-                        "FrameSourceVisitor.Visit() call")
-        self.assertEqual(g_client_handler.javascript_errors, 0,
-                         "Javascript errors caught in OnConsoleMessage")
-
-        # Javascript asserts
-        self.assertTrue(g_external.test_function_called,
-                        "js test_function() call")
-        self.assertTrue(g_external.test_callbacks_called,
-                        "js external.test_callbacks() call")
-        self.assertTrue(g_external.py_callback_called,
-                        "py_callback() call from js external.test_callbacks()")
+        # Client handler asserts and javascript External asserts
+        for obj in [g_client_handler, g_external]:
+            test_for_True = False  # Test whether asserts are working correctly
+            for key, value in obj.__dict__.items():
+                if key == "test_for_True":
+                    test_for_True = True
+                    continue
+                if "_True" in key:
+                    self.assertTrue(value, "Check assert: "+key)
+                    subtest_message(obj.__class__.__name__ + "." +
+                                    key.replace("_True", "") +
+                                    " ok")
+                elif "_False" in key:
+                    self.assertFalse(value, "Check assert: "+key)
+                    subtest_message(obj.__class__.__name__ + "." +
+                                    key.replace("_False", "") +
+                                    " ok")
+            self.assertTrue(test_for_True)
 
         # Test shutdown of CEF
         cef.Shutdown()
+        subtest_message("cef.Shutdown() ok")
 
+        # Display real number of tests there were run
+        print("\nRan " + str(g_subtests_ran) + " sub-tests in test_main")
+        sys.stdout.flush()
 
-class ClientHandler:
-    test_case = None
-
-    OnAfterCreated_called = False
-    OnLoadStart_called = False
-    OnLoadEnd_called = False
-
-    FrameSourceVisitor_called = False
-    frame_source_visitor = None
-
-    javascript_errors = 0
 
+class ClientHandler(object):
     def __init__(self, test_case):
         self.test_case = test_case
+        self.frame_source_visitor = None
+
+        # Asserts for True/False will be checked just before shutdown
+        self.test_for_True = True  # Test whether asserts are working correctly
+        self.OnAfterCreated_True = False
+        self.OnLoadStart_True = False
+        self.OnLoadEnd_True = False
+        self.FrameSourceVisitor_True = False
+        self.javascript_errors_False = False
+        self.OnConsoleMessage_True = False
 
     # noinspection PyUnusedLocal
     def _OnAfterCreated(self, browser):
-        self.OnAfterCreated_called = True
+        self.OnAfterCreated_True = True
 
     # noinspection PyUnusedLocal
     def OnLoadStart(self, browser, frame):
         self.test_case.assertEqual(browser.GetUrl(), g_datauri)
-        self.OnLoadStart_called = True
+        self.OnLoadStart_True = True
 
     # noinspection PyUnusedLocal
     def OnLoadEnd(self, browser, frame, http_code):
@@ -184,18 +210,21 @@ def OnLoadEnd(self, browser, frame, http_code):
         frame.GetSource(self.frame_source_visitor)
         browser.ExecuteJavascript(
                 "print('ClientHandler.OnLoadEnd() ok')")
-        self.OnLoadEnd_called = True
+        self.OnLoadEnd_True = True
 
     # noinspection PyUnusedLocal
     def OnConsoleMessage(self, browser, message, source, line):
         if "error" in message.lower() or "uncaught" in message.lower():
-            self.javascript_errors += 1
+            self.javascript_errors_False = True
             raise Exception(message)
+        else:
+            # Confirmation that messages from javascript are coming
+            self.OnConsoleMessage_True = True
+            subtest_message(message)
 
 
-class FrameSourceVisitor:
-    client_handler = None
-    test_case = None
+class FrameSourceVisitor(object):
+    """Visitor for Frame.GetSource()."""
 
     def __init__(self, client_handler, test_case):
         self.client_handler = client_handler
@@ -205,38 +234,39 @@ def __init__(self, client_handler, test_case):
     def Visit(self, value):
         self.test_case.assertIn("747ef3e6011b6a61e6b3c6e54bdd2dee",
                                 g_datauri_data)
-        self.client_handler.FrameSourceVisitor_called = True
+        self.client_handler.FrameSourceVisitor_True = True
 
 
-class External:
+class External(object):
     """Javascript 'window.external' object."""
-    test_case = None
-
-    # Test binding properties to the 'window' object.
-    test_property1 = "Test binding property to the 'window' object"
-    test_property2 = {"key1": test_property1,
-                      "key2": ["Inside list", 1, 2]}
-
-    test_function_called = False
-    test_callbacks_called = False
-    py_callback_called = False
 
     def __init__(self, test_case):
         self.test_case = test_case
 
+        # Test binding properties to the 'window' object.
+        self.test_property1 = "Test binding property to the 'window' object"
+        self.test_property2 = {"key1": self.test_property1,
+                               "key2": ["Inside list", 1, 2]}
+
+        # Asserts for True/False will be checked just before shutdown
+        self.test_for_True = True  # Test whether asserts are working correctly
+        self.test_function_True = False
+        self.test_callbacks_True = False
+        self.py_callback_True = False
+
     def test_function(self):
         """Test binding function to the 'window' object."""
-        self.test_function_called = True
+        self.test_function_True = True
 
     def test_callbacks(self, js_callback):
         """Test both javascript and python callbacks."""
         def py_callback(msg_from_js):
             self.test_case.assertEqual(msg_from_js,
                                        "String sent from Javascript")
-            self.py_callback_called = True
+            self.py_callback_True = True
         js_callback.Call("String sent from Python", py_callback)
-        self.test_callbacks_called = True
+        self.test_callbacks_True = True
 
 
 if __name__ == "__main__":
-    _runner.main(basename(__file__))
+    _test_runner.main(basename(__file__))
