make test failures inspectable; also, small cleanup

Technologicat · Technologicat · commit 4653e0741318 · 2020-09-04T03:52:23.000+03:00
diff --git a/unpythonic/syntax/testingtools.py b/unpythonic/syntax/testingtools.py
@@ -14,7 +14,7 @@
 
 from ..dynassign import dyn  # for MacroPy's gen_sym
 from ..env import env
-from ..misc import callsite_filename, safeissubclass
+from ..misc import callsite_filename
 from ..conditions import cerror, handlers, restarts, invoke
 from ..collections import unbox
 from ..symbol import sym, gensym
@@ -57,20 +57,17 @@ def istestmacro(tree):
 _error = sym("_error")  # used by the error[] macro
 _warn = sym("_warn")  # used by the warn[] macro
 
-_completed = sym("_completed")  # thunk returned normally
-_signaled = sym("_signaled")  # via unpythonic.conditions.signal and its sisters
-_raised = sym("_raised")  # via raise
 def _observe(thunk):
     """Run `thunk` and report how it fared.
 
     Internal helper for implementing assert functions.
 
     The return value is:
 
-      - `(_completed, return_value)` if the thunk completed normally
-      - `(_signaled, condition_instance)` if a signal from inside
+      - `(completed, return_value)` if the thunk completed normally
+      - `(signaled, condition_instance)` if a signal from inside
         the dynamic extent of thunk propagated to this level.
-      - `(_raised, exception_instance)` if an exception from inside
+      - `(raised, exception_instance)` if an exception from inside
         the dynamic extent of thunk propagated to this level.
     """
     def intercept(condition):
@@ -81,8 +78,7 @@ def intercept(condition):
         # it and let it fall through to the nearest enclosing `testset`, for
         # reporting. This can happen if a `test[]` is nested within a `with
         # test:` block, or if `test[]` expressions are nested.
-        exctype = type(condition)
-        if issubclass(exctype, fixtures.TestingException):
+        if issubclass(type(condition), fixtures.TestingException):
             return  # cancel and delegate to the next outer handler
         invoke("_got_signal", condition)
 
@@ -92,12 +88,12 @@ def intercept(condition):
                 ret = thunk()
             # We only reach this point if the restart was not invoked,
             # i.e. if thunk() completed normally.
-            return _completed, ret
-        return _signaled, unbox(sig)
+            return fixtures.completed, ret
+        return fixtures.signaled, unbox(sig)
     # This testing framework always signals, never raises, so we don't need any
     # special handling here.
     except Exception as err:  # including ControlError raised by an unhandled `unpythonic.conditions.error`
-        return _raised, err
+        return fixtures.raised, err
 
 
 _unassigned = gensym("_unassigned")  # runtime gensym / nonce value.
@@ -121,10 +117,11 @@ def unpythonic_assert(sourcecode, func, *, filename, lineno, message=None):
         `sourcecode` is a string representation of the source code expression
         that is being asserted.
 
-        `func` is the test itself, in the form a 1-argument function that
-        takes as its only argument an `unpythonic.env`. (The `the[]`
-        mechanism uses the `env` to store the value of the captured
-        subexpression.)
+        `func` is the test itself, as a 1-argument function that accepts
+        as its only argument an `unpythonic.env`. The `the[]` mechanism
+        uses this `env` to store the value of the captured subexpression.
+        (It is also perfectly fine to not store anything there; the presence
+         or absence of a captured value is detected automatically.)
 
         The function should compute the desired test expression and return
         its value. If the result is falsey, the assertion fails.
@@ -134,7 +131,7 @@ def unpythonic_assert(sourcecode, func, *, filename, lineno, message=None):
 
         `lineno` is the line number at the call site.
 
-        These are best extracted automatically using the `test[]` macro.
+        These are best extracted automatically using the test macros.
 
         `message` is an optional string, included in the generated error message
         if the assertion fails.
@@ -160,9 +157,11 @@ def unpythonic_assert(sourcecode, func, *, filename, lineno, message=None):
         custom_msg = ""
 
     # special cases for unconditional failures
-    if mode is _completed and test_result is _fail:  # fail[...], e.g. unreachable line reached
+    origin = "test"
+    if mode is fixtures.completed and test_result is _fail:  # fail[...], e.g. unreachable line reached
         fixtures._update(fixtures.tests_failed, +1)
         conditiontype = fixtures.TestFailure
+        origin = "fail"
         if message is not None:
             # If a user-given message is specified for `fail[]`, it is all
             # that should be displayed. We don't want confusing noise such as
@@ -172,18 +171,20 @@ def unpythonic_assert(sourcecode, func, *, filename, lineno, message=None):
             error_msg = message
         else:
             error_msg = "Unconditional failure requested, no message."
-    elif mode is _completed and test_result is _error:  # error[...], e.g. dependency not installed
+    elif mode is fixtures.completed and test_result is _error:  # error[...], e.g. dependency not installed
         fixtures._update(fixtures.tests_errored, +1)
         conditiontype = fixtures.TestError
+        origin = "error"
         if message is not None:
             error_msg = message
         else:
             error_msg = "Unconditional error requested, no message."
-    elif mode is _completed and test_result is _warn:  # warn[...], e.g. some test disabled for now
+    elif mode is fixtures.completed and test_result is _warn:  # warn[...], e.g. some test disabled for now
         fixtures._update(fixtures.tests_warned, +1)
         # HACK: warnings don't count into the test total
         fixtures._update(fixtures.tests_run, -1)
         conditiontype = fixtures.TestWarning
+        origin = "warn"
         if message is not None:
             error_msg = message
         else:
@@ -195,18 +196,18 @@ def unpythonic_assert(sourcecode, func, *, filename, lineno, message=None):
         #
         # So we may as well use the same code path as the fail and error cases.
     # general cases
-    elif mode is _completed:
+    elif mode is fixtures.completed:
         if test_result:
             return
         fixtures._update(fixtures.tests_failed, +1)
         conditiontype = fixtures.TestFailure
         error_msg = "Test failed: {}, due to result = {}{}".format(sourcecode, value, custom_msg)
-    elif mode is _signaled:
+    elif mode is fixtures.signaled:
         fixtures._update(fixtures.tests_errored, +1)
         conditiontype = fixtures.TestError
         desc = fixtures.describe_exception(test_result)
         error_msg = "Test errored: {}{}, due to unexpected signal: {}".format(sourcecode, custom_msg, desc)
-    else:  # mode is _raised:
+    else:  # mode is fixtures.raised:
         fixtures._update(fixtures.tests_errored, +1)
         conditiontype = fixtures.TestError
         desc = fixtures.describe_exception(test_result)
@@ -221,72 +222,79 @@ def unpythonic_assert(sourcecode, func, *, filename, lineno, message=None):
     # If the client code does not install a handler, then a `ControlError`
     # exception is raised by the condition system; leaving a cerror unhandled
     # is an error.
-    cerror(conditiontype(complete_msg))
+    #
+    # As well as forming an error message for humans, we provide the data
+    # in a machine-readable format for run-time inspection.
+    cerror(conditiontype(complete_msg, origin=origin, custom_message=message,
+                         filename=filename, lineno=lineno, sourcecode=sourcecode,
+                         mode=mode, result=test_result, captured_value=value))
 
 def unpythonic_assert_signals(exctype, sourcecode, thunk, *, filename, lineno, message=None):
     """Like `unpythonic_assert`, but assert that running `sourcecode` signals `exctype`.
 
     "Signal" as in `unpythonic.conditions.signal` and its sisters `error`, `cerror`, `warn`.
     """
-    mode, result = _observe(thunk)
+    mode, test_result = _observe(thunk)
     fixtures._update(fixtures.tests_run, +1)
 
     if message is not None:
         custom_msg = ", with message '{}'".format(message)
     else:
         custom_msg = ""
 
-    if mode is _completed:
+    if mode is fixtures.completed:
         fixtures._update(fixtures.tests_failed, +1)
         conditiontype = fixtures.TestFailure
         error_msg = "Test failed: {}{}, expected signal: {}, nothing was signaled.".format(sourcecode, custom_msg, fixtures.describe_exception(exctype))
-    elif mode is _signaled:
-        # allow both "signal(SomeError())" and "signal(SomeError)"
-        if isinstance(result, exctype) or safeissubclass(result, exctype):
+    elif mode is fixtures.signaled:
+        if isinstance(test_result, exctype):
             return
         fixtures._update(fixtures.tests_errored, +1)
         conditiontype = fixtures.TestError
-        desc = fixtures.describe_exception(result)
+        desc = fixtures.describe_exception(test_result)
         error_msg = "Test errored: {}{}, expected signal: {}, got unexpected signal: {}".format(sourcecode, custom_msg, fixtures.describe_exception(exctype), desc)
-    else:  # mode is _raised:
+    else:  # mode is fixtures.raised:
         fixtures._update(fixtures.tests_errored, +1)
         conditiontype = fixtures.TestError
-        desc = fixtures.describe_exception(result)
+        desc = fixtures.describe_exception(test_result)
         error_msg = "Test errored: {}{}, expected signal: {}, got unexpected exception: {}".format(sourcecode, custom_msg, fixtures.describe_exception(exctype), desc)
 
     complete_msg = "[{}:{}] {}".format(filename, lineno, error_msg)
-    cerror(conditiontype(complete_msg))
+    cerror(conditiontype(complete_msg, origin="test_signals", custom_message=message,
+                         filename=filename, lineno=lineno, sourcecode=sourcecode,
+                         mode=mode, result=test_result, captured_value=test_result))
 
 def unpythonic_assert_raises(exctype, sourcecode, thunk, *, filename, lineno, message=None):
     """Like `unpythonic_assert`, but assert that running `sourcecode` raises `exctype`."""
-    mode, result = _observe(thunk)
+    mode, test_result = _observe(thunk)
     fixtures._update(fixtures.tests_run, +1)
 
     if message is not None:
         custom_msg = ", with message '{}'".format(message)
     else:
         custom_msg = ""
 
-    if mode is _completed:
+    if mode is fixtures.completed:
         fixtures._update(fixtures.tests_failed, +1)
         conditiontype = fixtures.TestFailure
         error_msg = "Test failed: {}{}, expected exception: {}, nothing was raised.".format(sourcecode, custom_msg, fixtures.describe_exception(exctype))
-    elif mode is _signaled:
+    elif mode is fixtures.signaled:
         fixtures._update(fixtures.tests_errored, +1)
         conditiontype = fixtures.TestError
-        desc = fixtures.describe_exception(result)
+        desc = fixtures.describe_exception(test_result)
         error_msg = "Test errored: {}{}, expected exception: {}, got unexpected signal: {}".format(sourcecode, custom_msg, fixtures.describe_exception(exctype), desc)
-    else:  # mode is _raised:
-        # allow both "raise SomeError()" and "raise SomeError"
-        if isinstance(result, exctype) or safeissubclass(result, exctype):
+    else:  # mode is fixtures.raised:
+        if isinstance(test_result, exctype):
             return
         fixtures._update(fixtures.tests_errored, +1)
         conditiontype = fixtures.TestError
-        desc = fixtures.describe_exception(result)
+        desc = fixtures.describe_exception(test_result)
         error_msg = "Test errored: {}{}, expected exception: {}, got unexpected exception: {}".format(sourcecode, custom_msg, fixtures.describe_exception(exctype), desc)
 
     complete_msg = "[{}:{}] {}".format(filename, lineno, error_msg)
-    cerror(conditiontype(complete_msg))
+    cerror(conditiontype(complete_msg, origin="test_raises", custom_message=message,
+                         filename=filename, lineno=lineno, sourcecode=sourcecode,
+                         mode=mode, result=test_result, captured_value=test_result))
 
 
 # -----------------------------------------------------------------------------
diff --git a/unpythonic/test/fixtures.py b/unpythonic/test/fixtures.py
@@ -118,15 +118,18 @@
 
 from ..conditions import handlers, find_restart, invoke
 from ..collections import box, unbox
+from ..symbol import sym
 
 from .ansicolor import TC, colorize
 
 __all__ = ["session", "testset",
            "terminate",
            "returns_normally", "catch_signals",
            "TestConfig",
-           "tests_run", "tests_failed", "tests_errored",
-           "TestingException", "TestFailure", "TestError"]
+           "tests_run", "tests_failed", "tests_errored", "tests_warned",
+           "TestingException", "TestFailure", "TestError", "TestWarning",
+           "completed", "signaled", "raised",
+           "describe_exception"]
 
 # Keep a global count (since Python last started) of how many unpythonic_asserts
 # have run and how many have failed, so that the client code can easily calculate
@@ -169,18 +172,123 @@ def _reset(counter):
     with _counter_update_lock:
         counter << 0
 
+completed = sym("completed")
+completed.__doc__ = """TestingException `mode`: the test ran to completion normally.
+
+This does not mean that the test assertion succeeded, but only that
+it exited normally (i.e. did not signal or raise).
+"""
+signaled = sym("signaled")
+signaled.__doc__ = """TestingException `mode`: the test signaled a condition.
+
+The signal was not caught inside the test.
+
+See `unpythonic.conditions.signal` and its sisters `error`, `cerror`, `warn`.
+"""
+raised = sym("raised")
+raised.__doc__ = """TestingException `mode`: the test raised an exception.
+
+The exception was not caught inside the test.
+"""
 class TestingException(Exception):
     """Base type for testing-related exceptions."""
+    def __init__(self, *args, origin=None, custom_message=None,
+                 filename=None, lineno=None, sourcecode=None,
+                 mode=None, result=None, captured_value=None):
+        """Parameters:
+
+        `*args`: like in Exception. Usually just one, a human-readable
+                 error message as str.
+
+        Additionally, the test macros automatically fill in the following
+        optional parameters, for runtime inspection. These are stored in
+        instance attributes with the same name as the corresponding parameter:
+
+        `origin`: str, which of the test asserters produced this exception.
+                  One of "test", "test_signals", "test_raises", "fail", "error", "warn".
+
+        `custom_message`: str or None. The optional, user-provided human-readable
+                          custom failure message, if any.
+
+                          Test blocks that just assert that the block returns normally
+                          (default behavior if no `return` used) are encouraged to carry
+                          a clarifying message, provided by the user, to explain what was
+                          expected to happen, if it turns out that the test fails or errors.
+
+                          (Often, in such cases, the context alone is insufficient for a
+                           human not intimately familiar with the code to judge why the test
+                           block should or should not exit normally, so a message is useful.)
+
+                          Any invocation of `fail[]`, `error[]` and `warn[]` will also
+                          typically carry such a message, since that's the whole point
+                          of using those constructs.
+
+                          For any other use case though, the details are often clear enough
+                          from the code of the test assertion (or test block) itself, so
+                          there is no need for the user to include a custom failure message.
+
+        `filename`: str, the full path of the file containing the test in which
+                    the exception occurred.
+        `lineno`: int, line number in `filename` (1-based, as usual).
+        `sourcecode`: str, captured (actually unparsed from AST) source code of the test
+                      assertion (or test block). If a block, may have multiple lines.
+
+        `mode`: sym, how the test exited. One of `completed`, `signaled`, `raised` (which see).
+
+        `result`: If `mode is completed`, then the value of the test assertion (or the return
+                  value of a test block, respectively). Note test blocks that just assert
+                  that the block completes normally always return `True` when they complete
+                  normally.
+
+                  If `mode is signaled`, the signal instance (an `Exception` object).
+
+                  If `mode is raised`, the exception instance (an `Exception` object).
+
+                  If you need to format an exception (and its chained exceptions, if any)
+                  for human consumption, in a notation as close as possible to what Python
+                  itself uses for reporting uncaught exceptions, see `describe_exception`.
+
+        `captured_value`: The value the test macros refer to as "result" in the error messages.
+                If a `the[]` was used, the value of the `the[]` subexpression.
+
+                Else if the top level of the test assertion (or the return value of a test block,
+                respectively) was a comparison, the value of the leftmost term.
+
+                Else `captured_value is result`.
+
+                Note that `test_signals` and `test_raises` do not support capturing;
+                for them, it always holds that `captured_value is result`.
+        """
+        super().__init__(*args)
+        self.origin = origin
+        self.custom_message = custom_message
+        self.filename = filename
+        self.lineno = lineno
+        self.sourcecode = sourcecode
+        self.mode = mode
+        self.result = result
+        self.captured_value = captured_value
 class TestFailure(TestingException):
-    """Exception type indicating that a test failed."""
+    """Exception: a test ran to completion normally, but the test assertion failed.
+
+    May also mean that a test was expected to signal or raise, but it didn't.
+    """
 class TestError(TestingException):
-    """Exception type indicating that a test did not run to completion.
+    """Exception: a test did not run to completion normally.
+
+    It was also not expected to signal or raise, at least not the
+    exception type that was observed.
 
     This can happen due to an unexpected exception, or an unhandled
     `error` (or `cerror`) condition.
     """
 class TestWarning(TestingException):
-    """Exception type for a human-initiated test warning."""
+    """Exception: a human-initiated test warning.
+
+    Warnings (see the `warn[]` macro) can be used e.g. to mark tests that
+    are temporarily disabled due to external factors, such as language-level
+    compatibility issues, or a bug in a library yours depends on.
+    """
 
 def maybe_colorize(s, *colors):
     """Colorize `s` with ANSI color escapes if enabled in the global `TestConfig`.
