utPLSQL · jgebal · Feb 24, 2017 · Feb 4, 2017 · Feb 4, 2017 · Feb 5, 2017
diff --git a/docs/userguide/annotations.md b/docs/userguide/annotations.md
@@ -72,9 +72,20 @@ end test_pkg;
 | `%aftereach` | Procedure | Denotes that the annotated procedure should be executed after each `%test` method in the current suite. |
 | `%beforetest(<procedure_name>)` | Procedure | Denotes that mentioned procedure should be executed before the annotated `%test` procedure. |
 | `%aftertest(<procedure_name>)` | Procedure | Denotes that mentioned procedure should be executed after the annotated `%test` procedure. |
-| `%rollback(<type>)` | Package/procedure | Configure transaction control behaviour (type). Supported values: `auto`(default) - rollback to savepoint (before the test/suite setup) is issued after each test/suite teardown; `manual` - rollback is never issued automatically. Property can be overridden for child element (test in suite) |
+| `%rollback(<type>)` | Package/procedure | Configure transaction control behaviour (type). Supported values: `auto`(default) - A savepoint is created before invocation of each "before block" is and a rollback to specific savepoint is issued after each "after" block; `manual` - rollback is never issued automatically. Property can be overridden for child element (test in suite) |
 | `%disabled` | Package/procedure | Used to disable a suite or a test |
 
+# Using automatic rollbacks in tests
+By default, every test is isolated from other tests using savepoint.
+This solution is suitable for use-cases, where the code that is getting tested as well as the unit tests themselves do not use transaction control commands (commit/rollback).
+In general, your unit tests should not use transaction control as long as the core you are testing is not using it too.
+Keeping the transactions uncommitted allows your changes to be isolated and the execution of tests is not impacting others that might be using a shared (integration) development database.
+
+If however you're in situation, where the code you are testing, is using transaction control (like ETL code is usually doing), then your tests should not use the default rollback(auto)
+You should make sure that thr entire suitepath all the way to the root is using manual transaction control in that case.
+
+In some cases it is needed to perform DDL in setup/teardown. It is recommended to move such DDL statements to a procedure with pragma autonomous_transaction to eliminate implicit commit of the main session.
+
 # Suitepath concept
 It is very likely that the application for which you are going to introduce tests consists of many different packages or procedures/functions. Usually procedures can be logically grouped inside a package, there also might be several logical groups of procedure in a single package or even packages themselves might relate to a common module.
 
@@ -148,3 +159,4 @@ A `%suitepath` can be provided in tree ways:
 * schema - execute all test in the schema
 * [schema]:suite1[.suite2][.suite3]...[.procedure] - execute all tests in all suites from suite1[.suite2][.suite3]...[.procedure] path. If schema is not provided, then current schema is used. Example: `:all.rooms_tests`.
 * [schema.]package[.procedure] - execute all tests in the test package provided. The whole hierarchy of suites in the schema is build before, all before/after hooks of partn suites for th provided suite package are executed as well. Example: `tests.test_contact.test_last_name_validator` or simply `test_contact.test_last_name_validator` if `tests` is the current schema.
+
diff --git a/docs/userguide/exception-reporting.md b/docs/userguide/exception-reporting.md
@@ -0,0 +1,121 @@
+# Exception handling and reporting
+
+The utPLSQL is responsible for handling exceptions wherever they occur in the test run. utPLSQL is trapping most of the exceptions so that the test execution is not affected by individual tests or test packages throwing an exception.
+The framework provides a full stacktrace for every exception that was thrown. The stacktrace is clean and does not include any utPLSQL library calls in it.
+To achieve rerunability, the ORA-04068, ORA-04061 exceptions are not handled and test execution will be interrupted if such exception is encountered. This is because of how Oracle behaves on those exceptions.
+
+Test execution can fail for different reasons. The failures on different exceptions are handled as follows:
+* A test package without body - each `%test` is reported as failed with exception, nothing is executed
+* A test package with _invalid body_ - each `%test` is reported as failed with exception, nothing is executed
+* A test package with _invalid spec_ - package is not considered a valid unit test package and is excluded from execution. When trying to run a test package with invalid spec explicitly, exception is raised. Only valid specifications are parsed for annotations 
+* A test package that is raising an exception in `%beforeall` - each `%test` is reported as failed with exception, `%test`, `%beforeeach`, `%beforetest`, `%aftertest` and `%aftereach` are not executed. `%afterall` is executed to allow cleanup of whatever was done in `%beforeall`
+* A test package that is raising an exception in `%beforeeach` - each `%test` is reported as failed with exception, `%test`, `%beforetest` and `%aftertest` is not executed. The `%aftereach` and `%afterall` blocks are getting executed to allow cleanup of whatever was done in `%before...` blocks
+* A test package that is raising an exception in `%beforetest` - the `%test` is reported as failed  with exception, `%test` is not executed. The `%aftertest`, `%aftereach` and `%afterall` blocks are getting executed to allow cleanup of whatever was done in `%before...` blocks
+* A test package that is raising an exception in `%test` - the `%test` is reported as failed with exception. The execution of other blocks continues normally
+* A test package that is raising an exception in `%aftertest` - the `%test` is reported as failed with exception. The execution of other blocks continues normally
+* A test package that is raising an exception in `%aftereach` - all blocks of the package are executed, as ehe `%aftereach` is a closing block for an individual test. Exception in `%aftereach` is not affecting test results. For every failed execution of `%aftereach` a warning with exception stacktrace is displayed in the summary
+* A test package that is raising an exception in `%afterall` - all blocks of  the package are executed, as the `%afterall` is the last step of package execution. Exception in `%afterall` is not affecting test results. A warning with exception stacktrace is displayed in the summary
+
+
+Example of reporting with exception thrown in `%beforetest`:
+````
+Remove rooms by name
+  Removes a room without content in it (FAILED - 1)
+  Does not remove room when it has content
+  Raises exception when null room name given
+
+Failures:
+
+  1) remove_empty_room
+
+        error: ORA-20001: Test exception
+               ORA-06512: at "UT3.TEST_REMOVE_ROOMS_BY_NAME", line 39
+               ORA-06512: at line 6
+
+Finished in ,039346 seconds
+3 tests, 0 failed, 1 errored, 0 ignored.
+````
+
+Example of reporting with exception thrown in `%test`:
+```
+Remove rooms by name
+  Removes a room without content in it (FAILED - 1)
+  Does not remove room when it has content
+  Raises exception when null room name given
+
+Failures:
+
+  1) remove_empty_room
+
+        error: ORA-20001: Test exception
+               ORA-06512: at "UT3.TEST_REMOVE_ROOMS_BY_NAME", line 48
+               ORA-06512: at line 6
+
+Finished in ,035726 seconds
+3 tests, 0 failed, 1 errored, 0 ignored.
+```
+
+Example of reporting with exception thrown in `%aftertest`:
+```
+Remove rooms by name
+  Removes a room without content in it (FAILED - 1)
+  Does not remove room when it has content
+  Raises exception when null room name given
+
+Failures:
+
+  1) remove_empty_room
+
+        error: ORA-20001: Test exception
+               ORA-06512: at "UT3.TEST_REMOVE_ROOMS_BY_NAME", line 42
+               ORA-06512: at line 6
+
+Finished in ,045523 seconds
+3 tests, 0 failed, 1 errored, 0 ignored.
+```
+
+Example of reporting with exception thrown in `%aftereach`:
+```
+Remove rooms by name
+  Removes a room without content in it
+  Does not remove room when it has content
+  Raises exception when null room name given
+
+Warnings:
+
+  1) test_remove_rooms_by_name - Aftereach procedure failed:
+       ORA-20001: Test exception
+       ORA-06512: at "UT3.TEST_REMOVE_ROOMS_BY_NAME", line 31
+       ORA-06512: at line 6
+
+  2) test_remove_rooms_by_name - Aftereach procedure failed:
+       ORA-20001: Test exception
+       ORA-06512: at "UT3.TEST_REMOVE_ROOMS_BY_NAME", line 31
+       ORA-06512: at line 6
+
+  3) test_remove_rooms_by_name - Aftereach procedure failed:
+       ORA-20001: Test exception
+       ORA-06512: at "UT3.TEST_REMOVE_ROOMS_BY_NAME", line 31
+       ORA-06512: at line 6
+
+Finished in ,05071 seconds
+3 tests, 0 failed, 0 errored, 0 ignored. 3 warning(s)
+```
+
+Example of reporting with exception thrown in `%afterall`:
+```
+Remove rooms by name
+  Removes a room without content in it
+  Does not remove room when it has content
+  Raises exception when null room name given
+
+Warnings:
+
+  1) test_remove_rooms_by_name - Afterall procedure failed: 
+       ORA-20001: Test exception
+       ORA-06512: at "UT3.TEST_REMOVE_ROOMS_BY_NAME", line 35
+       ORA-06512: at line 6
+
+Finished in ,044902 seconds
+3 tests, 0 failed, 0 errored, 0 ignored. 1 warning(s)
+```
diff --git a/source/core/types/ut_event_listener.tpb b/source/core/types/ut_event_listener.tpb
@@ -88,7 +88,7 @@ create or replace type body ut_event_listener is
       end if;
     end loop;
 
-  end;
+  end fire_event;
 
 end;
 /
diff --git a/source/core/types/ut_logical_suite.tpb b/source/core/types/ut_logical_suite.tpb
@@ -52,39 +52,35 @@ create or replace type body ut_logical_suite as
     self.items(self.items.last) := a_item;
   end;
 
-  overriding member procedure do_execute(self in out nocopy ut_logical_suite, a_listener in out nocopy ut_event_listener_base) is
-    l_completed_without_errors boolean;
-  begin
-    l_completed_without_errors := self.do_execute(a_listener);
-  end;
-
   overriding member function do_execute(self in out nocopy ut_logical_suite, a_listener in out nocopy ut_event_listener_base) return boolean is
     l_suite_savepoint varchar2(30);
     l_item_savepoint  varchar2(30);
     l_completed_without_errors boolean;
   begin
     ut_utils.debug_log('ut_logical_suite.execute');
+
+    a_listener.fire_before_event(ut_utils.gc_suite,self);   
+    self.start_time := current_timestamp; 
 
     if self.get_ignore_flag() then
       self.result := ut_utils.tr_ignore;
+      self.end_time := self.start_time;
       ut_utils.debug_log('ut_logical_suite.execute - ignored');
     else
-      a_listener.fire_before_event(ut_utils.gc_suite,self);
 
       self.start_time := current_timestamp;
 
       for i in 1 .. self.items.count loop
         -- execute the item (test or suite)
         self.items(i).do_execute(a_listener);
-
       end loop;
 
       self.calc_execution_result();
-
       self.end_time := current_timestamp;
 
-      a_listener.fire_after_event(ut_utils.gc_suite,self);
     end if;
+
+    a_listener.fire_after_event(ut_utils.gc_suite,self);
 
     return l_completed_without_errors;
   end;
@@ -103,7 +99,21 @@ create or replace type body ut_logical_suite as
     end if;
 
       self.result := l_result;
-    end;
+  end;
+
+  overriding member procedure fail(self in out nocopy ut_logical_suite, a_listener in out nocopy ut_event_listener_base, a_failure_msg varchar2) is
+  begin
+    ut_utils.debug_log('ut_logical_suite.fail');
+    a_listener.fire_before_event(ut_utils.gc_suite, self);
+    self.start_time := current_timestamp;
+    for i in 1 .. self.items.count loop
+      -- execute the item (test or suite)
+      self.items(i).fail(a_listener,a_failure_msg);
+    end loop;
+    self.calc_execution_result();
+    self.end_time := self.start_time;
+    a_listener.fire_after_event(ut_utils.gc_suite, self);
+  end;  
 
 end;
 /
diff --git a/source/core/types/ut_logical_suite.tps b/source/core/types/ut_logical_suite.tps
@@ -1,4 +1,4 @@
-create or replace type ut_logical_suite force under ut_suite_item (
+create or replace type ut_logical_suite under ut_suite_item (
   /*
   utPLSQL - Version X.X.X.X
   Copyright 2016 - 2017 utPLSQL Project
@@ -31,7 +31,7 @@ create or replace type ut_logical_suite force under ut_suite_item (
   member function item_index(a_name varchar2) return pls_integer,
   member procedure add_item(self in out nocopy ut_logical_suite, a_item ut_suite_item),
   overriding member function  do_execute(self in out nocopy ut_logical_suite, a_listener in out nocopy ut_event_listener_base) return boolean,
-  overriding member procedure do_execute(self in out nocopy ut_logical_suite, a_listener in out nocopy ut_event_listener_base),
-  overriding member procedure calc_execution_result(self in out nocopy ut_logical_suite)
+  overriding member procedure calc_execution_result(self in out nocopy ut_logical_suite),
+  overriding member procedure fail(self in out nocopy ut_logical_suite, a_listener in out nocopy ut_event_listener_base, a_failure_msg varchar2)
 ) not final
 /
diff --git a/source/core/types/ut_reporter_base.tpb b/source/core/types/ut_reporter_base.tpb
@@ -131,5 +131,6 @@ create or replace type body ut_reporter_base is
   begin
     ut_output_buffer.close(self);
   end;
+
 end;
 /
diff --git a/source/core/types/ut_reporter_base.tps b/source/core/types/ut_reporter_base.tps
@@ -1,4 +1,4 @@
-create or replace type ut_reporter_base force authid current_user as object(
+create or replace type ut_reporter_base authid current_user as object(
   /*
   utPLSQL - Version X.X.X.X
   Copyright 2016 - 2017 utPLSQL Project

diff --git a/source/core/types/ut_results_counter.tpb b/source/core/types/ut_results_counter.tpb
@@ -17,10 +17,11 @@ create or replace type body ut_results_counter as
   */
   constructor function ut_results_counter(self in out nocopy ut_results_counter) return self as result is
   begin
-    self.ignored_count := 0;
-    self.success_count := 0;
-    self.failure_count := 0;
-    self.errored_count := 0;
+    self.ignored_count  := 0;
+    self.success_count  := 0;
+    self.failure_count  := 0;
+    self.errored_count  := 0;
+    self.warnings_count := 0;
     return;
   end;
 
@@ -39,10 +40,17 @@ create or replace type body ut_results_counter as
     self.success_count  := self.success_count + a_item.success_count;
     self.failure_count  := self.failure_count + a_item.failure_count;
     self.errored_count  := self.errored_count + a_item.errored_count;
+    self.warnings_count := self.warnings_count + a_item.warnings_count;
+  end;
+
+  member procedure increase_warning_count(self in out nocopy ut_results_counter) is
+  begin
+    self.warnings_count := self.warnings_count + 1;
   end;
 
   member function total_count return integer is
   begin
+    --skip warnings here
     return self.ignored_count + self.success_count + self.failure_count + self.errored_count;
   end;
 

diff --git a/source/core/types/ut_results_counter.tps b/source/core/types/ut_results_counter.tps
@@ -19,9 +19,11 @@ create or replace type ut_results_counter as object(
   success_count integer,
   failure_count integer,
   errored_count integer,
+  warnings_count integer,
   constructor function ut_results_counter(self in out nocopy ut_results_counter) return self as result,
   constructor function ut_results_counter(self in out nocopy ut_results_counter, a_status integer) return self as result,
   member procedure sum_counter_values(self in out nocopy ut_results_counter, a_item ut_results_counter),
+  member procedure increase_warning_count(self in out nocopy ut_results_counter),
   member function total_count return integer,
   member function result_status return integer
 )

diff --git a/source/core/types/ut_run.tpb b/source/core/types/ut_run.tpb
@@ -24,12 +24,6 @@ create or replace type body ut_run as
     return;
   end;
 
-  overriding member procedure do_execute(self in out nocopy ut_run, a_listener in out nocopy ut_event_listener_base) is
-    l_completed_without_errors boolean;
-  begin
-    l_completed_without_errors := self.do_execute(a_listener);
-  end;
-
   overriding member function do_execute(self in out nocopy ut_run, a_listener in out nocopy ut_event_listener_base) return boolean is
     l_completed_without_errors boolean;
   begin
@@ -38,6 +32,9 @@ create or replace type body ut_run as
     a_listener.fire_before_event(ut_utils.gc_run, self);
 
     self.start_time := current_timestamp;
+
+    -- clear anything that might stay in the session's cache
+    ut_assert_processor.clear_asserts;
 
     for i in 1 .. self.items.count loop
       l_completed_without_errors := self.items(i).do_execute(a_listener);
@@ -67,6 +64,23 @@ create or replace type body ut_run as
 
     self.result := l_result;
   end;
+
+  overriding member procedure fail(self in out nocopy ut_run, a_listener in out nocopy ut_event_listener_base, a_failure_msg varchar2) is
+  begin
+    ut_utils.debug_log('ut_run.fail');
+
+    a_listener.fire_before_event(ut_utils.gc_run, self);
+    self.start_time := current_timestamp;
+
+    for i in 1 .. self.items.count loop
+      self.items(i).fail(a_listener, a_failure_msg);
+    end loop;
+
+    self.calc_execution_result();
+    self.end_time := self.start_time;
+
+    a_listener.fire_after_event(ut_utils.gc_run, self);
+  end;
 
 end;
 /
diff --git a/source/core/types/ut_run.tps b/source/core/types/ut_run.tps
@@ -1,4 +1,4 @@
-create or replace type ut_run authid current_user under ut_suite_item (
+create or replace type ut_run under ut_suite_item (
   /*
   utPLSQL - Version X.X.X.X
   Copyright 2016 - 2017 utPLSQL Project
@@ -21,7 +21,7 @@ create or replace type ut_run authid current_user under ut_suite_item (
   items        ut_suite_items,
   constructor function ut_run( self in out nocopy ut_run, a_items ut_suite_items ) return self as result,
   overriding member function  do_execute(self in out nocopy ut_run, a_listener in out nocopy ut_event_listener_base) return boolean,
-  overriding member procedure do_execute(self in out nocopy ut_run, a_listener in out nocopy ut_event_listener_base),
-  overriding member procedure calc_execution_result(self in out nocopy ut_run)
+  overriding member procedure calc_execution_result(self in out nocopy ut_run),
+  overriding member procedure fail(self in out nocopy ut_run, a_listener in out nocopy ut_event_listener_base, a_failure_msg varchar2)
 )
 /
-Original file line number
+Diff line change
@@ Expand Up / @@ -88,7 +88,7 @@ create or replace type body ut_event_listener is @@
           end if;
         end loop;
-      end;
+      end fire_event;
     end;
     /