diff --git a/development/cleanup.sh b/development/cleanup.sh index 3b6e22e53..989be30b9 100755 --- a/development/cleanup.sh +++ b/development/cleanup.sh @@ -7,6 +7,7 @@ git rev-parse && cd "$(git rev-parse --show-cdup)" "${SQLCLI}" sys/${ORACLE_PWD}@//${CONNECTION_STR} AS SYSDBA <<-SQL set echo on +set serveroutput on begin for x in ( select * from dba_objects @@ -25,12 +26,12 @@ drop user ${UT3_USER} cascade; begin for i in ( - select decode(owner,'PUBLIC','drop public synonym "','drop synonym "'||owner||'"."')|| synonym_name ||'"' drop_orphaned_synonym from dba_synonyms a + select decode(owner,'PUBLIC','drop public synonym "','drop synonym "'||owner||'"."')|| synonym_name ||'"' drop_orphaned_synonym, owner||'.'||synonym_name syn from dba_synonyms a where not exists (select 1 from dba_objects b where (a.table_name=b.object_name and a.table_owner=b.owner or b.owner='SYS' and a.table_owner=b.object_name) ) and a.table_owner not in ('SYS','SYSTEM') ) loop - dbms_output.put_line(i.drop_orphaned_synonym); execute immediate i.drop_orphaned_synonym; + dbms_output.put_line('synonym '||i.syn||' dropped'); end loop; end; / diff --git a/docs/userguide/running-unit-tests.md b/docs/userguide/running-unit-tests.md index 349bb0a05..a6482023d 100644 --- a/docs/userguide/running-unit-tests.md +++ b/docs/userguide/running-unit-tests.md @@ -47,6 +47,7 @@ The examples below illustrate different ways and options to invoke `ut.run` proc ```sql alter session set current_schema=hr; +set serveroutput on begin ut.run(); end; @@ -55,6 +56,7 @@ Executes all tests in current schema (_HR_). ```sql +set serveroutput on begin ut.run('HR'); end; @@ -63,6 +65,7 @@ Executes all tests in specified schema (_HR_). ```sql +set serveroutput on begin ut.run('hr:com.my_org.my_project'); end; @@ -73,6 +76,7 @@ Check the [annotations documentation](annotations.md) to find out about suitepat ```sql +set serveroutput on begin ut.run('hr.test_apply_bonus'); end; @@ -81,6 +85,7 @@ Executes all tests from package _hr.test_apply_bonus_. ```sql +set serveroutput on begin ut.run('hr.test_apply_bonus.bonus_cannot_be_negative'); end; @@ -89,6 +94,7 @@ Executes single test procedure _hr.test_apply_bonus.bonus_cannot_be_negative_. ```sql +set serveroutput on begin ut.run(ut_varchar2_list('hr.test_apply_bonus','cust')); end; @@ -96,6 +102,7 @@ end; Executes all tests from package _hr.test_apply_bonus_ and all tests from schema _cust_. ```sql +set serveroutput on begin ut.run(ut_varchar2_list('hr.test_apply_bonus,cust)'); end; @@ -104,6 +111,7 @@ end; Executes all tests from package _hr.test_apply_bonus_ and all tests from schema _cust_. ```sql +set serveroutput on begin ut.run('hr.test_apply_bonus,cust'); end; @@ -124,6 +132,7 @@ The `ut.run` procedures and functions accept `a_reporter` attribute that defines You can execute any set of tests with any of the predefined reporters. ```sql +set serveroutput on begin ut.run('hr.test_apply_bonus', ut_junit_reporter()); end; @@ -133,26 +142,6 @@ Executes all tests from package _HR.TEST_APPLY_BONUS_ and provide outputs to DBM For details on build-in reporters look at [reporters documentation](reporters.md). -## Keeping uncommited data after test-run - -utPLSQL by default runs tests in autonomous transaction and performs automatic rollback to assure that tests do not impact one-another and do not have impact on the current session in your IDE. - -If you would like to keep your uncommited data persisted after running tests, you can do so by using `a_force_manual_rollback` flag. -Setting this flag to true has following side-effects: - -- test execution is done in current transaction - if while running tests commit or rollback is issued your current session data will get commited too. -- automatic rollback is forced to be disabled in test-run even if it was explicitly enabled by using annotation `--%rollback(manual) - -Example invocation: -```sql -begin - ut.run('hr.test_apply_bonus', a_force_manual_rollback => true); -end; -``` - - -This option is not anvailable when running tests using `ut.run` as a table function. - ## ut.run functions The `ut.run` functions provide exactly the same functionality as the `ut.run` procedures. @@ -160,6 +149,10 @@ You may use the same sets of parameters with both functions and procedures. The only difference is the output of the results. Functions provide output as a pipelined stream and therefore need to be executed as select statements. +**Note:** +>When running tests with `ut.run` functions, whole test run is executed as autonomous transaction. +At the end of the run, the transaction is automatically rolled-back and all uncommitted changes are reverted. + Example. ```sql select * from table(ut.run('hr.test_apply_bonus', ut_junit_reporter())); @@ -180,7 +173,83 @@ The concept is pretty simple. - as a separate thread, start `ut_runner.run` and pass reporters with previously defined output_ids. - for each reporter start a separate thread and read outputs from the `ut_output_buffer.get_lines` table function by providing the output_id defined in the main thread. -# Reports characterset encoding +# Order of test execution + +## Default order + +When unit tests are executed without random order, they are ordered by: +- schema name +- suite path or test package name if `--%suitepath` was not specified for that package +- `--%test` line number in package + +## Random order + +You can force a test run to execute tests in random order by providing one of options to `ut.run`: +- `a_random_test_order` - true/false for procedures and 1/0 for functions +- `a_random_test_order_seed` - positive number in range of 1 .. 1 000 000 000 + +When tests are executed with random order, randomization is applied to single level of suitepath hierarchy tree. +This is needed to maintain visibility and accessibility of common setup/cleanup `beforeall`/`afterall` in tests. + +Example: +```sql +set serveroutput on +begin + ut.run('hr.test_apply_bonus', a_random_test_order => true); +end; +``` + +```sql +select * from table(ut.run('hr.test_apply_bonus', a_random_test_order => 1)); +``` + +When running with random order, the default report (`ut_documentation_reporter`) will include information about the random test run seed. +Example output: +``` +... +Finished in .12982 seconds +35 tests, 0 failed, 0 errored, 1 disabled, 0 warning(s) +Tests were executed with random order seed '302980531'. +``` + +If you want to re-run tests using previously generated seed, you may do so by running them with parameter `a_random_test_order_seed` +Example: +```sql +set serveroutput on +begin + ut.run('hr.test_apply_bonus', a_random_test_order_seed => 302980531); +end; +``` + +```sql +select * from table(ut.run('hr.test_apply_bonus', a_random_test_order_seed => 302980531)); +``` + +**Note** +>Random order seed must be a positive number within range of 1 .. 1 000 000 000. + +# Keeping uncommitted data after test-run + +utPLSQL by default runs tests in autonomous transaction and performs automatic rollback to assure that tests do not impact one-another and do not have impact on the current session in your IDE. + +If you would like to keep your uncommitted data persisted after running tests, you can do so by using `a_force_manual_rollback` flag. +Setting this flag to true has following side-effects: + +- test execution is done in current transaction - if while running tests commit or rollback is issued your current session data will get commited too. +- automatic rollback is forced to be disabled in test-run even if it was explicitly enabled by using annotation `--%rollback(manual) + +Example invocation: +```sql +set serveroutput on +begin + ut.run('hr.test_apply_bonus', a_force_manual_rollback => true); +end; +``` + +**Note:** +>This option is not available when running tests using `ut.run` as a table function. + +# Reports character-set encoding To get properly encoded reports, when running utPLSQL with HTML/XML reports on data containing national characters you need to provide your client character set when calling `ut.run` functions and procedures. diff --git a/source/api/ut.pkb b/source/api/ut.pkb index eb4a06bc3..5e13dcc62 100644 --- a/source/api/ut.pkb +++ b/source/api/ut.pkb @@ -113,12 +113,14 @@ create or replace package body ut is a_paths ut_varchar2_list, a_reporter in out nocopy ut_reporter_base, a_color_console integer, - a_coverage_schemes ut_varchar2_list := null, + a_coverage_schemes ut_varchar2_list, a_source_file_mappings ut_file_mappings, a_test_file_mappings ut_file_mappings, a_include_objects ut_varchar2_list, a_exclude_objects ut_varchar2_list, - a_client_character_set varchar2 := null + a_client_character_set varchar2, + a_random_test_order integer, + a_random_test_order_seed positive ) is pragma autonomous_transaction; begin @@ -133,7 +135,10 @@ create or replace package body ut is a_include_objects, a_exclude_objects, gc_fail_on_errors, - a_client_character_set + a_client_character_set, + false, + ut_utils.int_to_boolean(a_random_test_order), + a_random_test_order_seed ); rollback; end; @@ -142,12 +147,14 @@ create or replace package body ut is a_paths ut_varchar2_list, a_reporter in out nocopy ut_reporter_base, a_color_console integer, - a_coverage_schemes ut_varchar2_list := null, + a_coverage_schemes ut_varchar2_list, a_source_files ut_varchar2_list, a_test_files ut_varchar2_list, a_include_objects ut_varchar2_list, a_exclude_objects ut_varchar2_list, - a_client_character_set varchar2 := null + a_client_character_set varchar2, + a_random_test_order integer, + a_random_test_order_seed positive ) is pragma autonomous_transaction; begin @@ -162,7 +169,10 @@ create or replace package body ut is a_include_objects, a_exclude_objects, gc_fail_on_errors, - a_client_character_set + a_client_character_set, + false, + ut_utils.int_to_boolean(a_random_test_order), + a_random_test_order_seed ); rollback; end; @@ -200,7 +210,9 @@ create or replace package body ut is a_test_file_mappings ut_file_mappings := null, a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, - a_client_character_set varchar2 := null + a_client_character_set varchar2 := null, + a_random_test_order integer := 0, + a_random_test_order_seed positive := null ) return ut_varchar2_rows pipelined is l_reporter ut_reporter_base := a_reporter; l_results sys_refcursor; @@ -214,7 +226,9 @@ create or replace package body ut is a_test_file_mappings, a_include_objects, a_exclude_objects, - a_client_character_set + a_client_character_set, + a_random_test_order, + a_random_test_order_seed ); if l_reporter is of (ut_output_reporter_base) then l_results := treat(l_reporter as ut_output_reporter_base).get_lines_cursor(); @@ -233,7 +247,9 @@ create or replace package body ut is a_test_files ut_varchar2_list, a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, - a_client_character_set varchar2 := null + a_client_character_set varchar2 := null, + a_random_test_order integer := 0, + a_random_test_order_seed positive := null ) return ut_varchar2_rows pipelined is l_reporter ut_reporter_base := a_reporter; l_results sys_refcursor; @@ -247,7 +263,9 @@ create or replace package body ut is a_test_files, a_include_objects, a_exclude_objects, - a_client_character_set + a_client_character_set, + a_random_test_order, + a_random_test_order_seed ); if l_reporter is of (ut_output_reporter_base) then l_results := treat(l_reporter as ut_output_reporter_base).get_lines_cursor(); @@ -267,7 +285,9 @@ create or replace package body ut is a_test_file_mappings ut_file_mappings := null, a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, - a_client_character_set varchar2 := null + a_client_character_set varchar2 := null, + a_random_test_order integer := 0, + a_random_test_order_seed positive := null ) return ut_varchar2_rows pipelined is l_reporter ut_reporter_base := a_reporter; l_results sys_refcursor; @@ -281,7 +301,9 @@ create or replace package body ut is a_test_file_mappings, a_include_objects, a_exclude_objects, - a_client_character_set + a_client_character_set, + a_random_test_order, + a_random_test_order_seed ); if l_reporter is of (ut_output_reporter_base) then l_results := treat(l_reporter as ut_output_reporter_base).get_lines_cursor(); @@ -301,7 +323,9 @@ create or replace package body ut is a_test_files ut_varchar2_list, a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, - a_client_character_set varchar2 := null + a_client_character_set varchar2 := null, + a_random_test_order integer := 0, + a_random_test_order_seed positive := null ) return ut_varchar2_rows pipelined is l_reporter ut_reporter_base := a_reporter; l_results sys_refcursor; @@ -315,7 +339,9 @@ create or replace package body ut is a_test_files, a_include_objects, a_exclude_objects, - a_client_character_set + a_client_character_set, + a_random_test_order, + a_random_test_order_seed ); if l_reporter is of (ut_output_reporter_base) then l_results := treat(l_reporter as ut_output_reporter_base).get_lines_cursor(); @@ -335,7 +361,9 @@ create or replace package body ut is a_test_file_mappings ut_file_mappings := null, a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, - a_client_character_set varchar2 := null + a_client_character_set varchar2 := null, + a_random_test_order integer := 0, + a_random_test_order_seed positive := null ) return ut_varchar2_rows pipelined is l_reporter ut_reporter_base := a_reporter; l_results sys_refcursor; @@ -349,7 +377,9 @@ create or replace package body ut is a_test_file_mappings, a_include_objects, a_exclude_objects, - a_client_character_set + a_client_character_set, + a_random_test_order, + a_random_test_order_seed ); if l_reporter is of (ut_output_reporter_base) then l_results := treat(l_reporter as ut_output_reporter_base).get_lines_cursor(); @@ -369,7 +399,9 @@ create or replace package body ut is a_test_files ut_varchar2_list, a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, - a_client_character_set varchar2 := null + a_client_character_set varchar2 := null, + a_random_test_order integer := 0, + a_random_test_order_seed positive := null ) return ut_varchar2_rows pipelined is l_reporter ut_reporter_base := a_reporter; l_results sys_refcursor; @@ -383,7 +415,9 @@ create or replace package body ut is a_test_files, a_include_objects, a_exclude_objects, - a_client_character_set + a_client_character_set, + a_random_test_order, + a_random_test_order_seed ); if l_reporter is of (ut_output_reporter_base) then l_results := treat(l_reporter as ut_output_reporter_base).get_lines_cursor(); @@ -404,7 +438,9 @@ create or replace package body ut is a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ) is l_reporter ut_reporter_base := a_reporter; begin @@ -421,7 +457,9 @@ create or replace package body ut is a_exclude_objects, gc_fail_on_errors, a_client_character_set, - a_force_manual_rollback + a_force_manual_rollback, + a_random_test_order, + a_random_test_order_seed ); else run_autonomous( @@ -433,7 +471,9 @@ create or replace package body ut is a_test_file_mappings, a_include_objects, a_exclude_objects, - a_client_character_set + a_client_character_set, + ut_utils.boolean_to_int(a_random_test_order), + a_random_test_order_seed ); end if; if l_reporter is of (ut_output_reporter_base) then @@ -452,7 +492,9 @@ create or replace package body ut is a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ) is l_reporter ut_reporter_base := a_reporter; begin @@ -466,7 +508,9 @@ create or replace package body ut is a_include_objects, a_exclude_objects, a_client_character_set, - a_force_manual_rollback + a_force_manual_rollback, + a_random_test_order, + a_random_test_order_seed ); end; @@ -479,7 +523,9 @@ create or replace package body ut is a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ) is begin ut.run( @@ -492,7 +538,9 @@ create or replace package body ut is a_include_objects, a_exclude_objects, a_client_character_set, - a_force_manual_rollback + a_force_manual_rollback, + a_random_test_order, + a_random_test_order_seed ); end; @@ -505,7 +553,9 @@ create or replace package body ut is a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ) is begin ut.run( @@ -518,7 +568,9 @@ create or replace package body ut is a_include_objects, a_exclude_objects, a_client_character_set, - a_force_manual_rollback + a_force_manual_rollback, + a_random_test_order, + a_random_test_order_seed ); end; @@ -532,7 +584,9 @@ create or replace package body ut is a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ) is begin ut.run( @@ -545,7 +599,9 @@ create or replace package body ut is a_include_objects, a_exclude_objects, a_client_character_set, - a_force_manual_rollback + a_force_manual_rollback, + a_random_test_order, + a_random_test_order_seed ); end; @@ -559,7 +615,9 @@ create or replace package body ut is a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ) is begin ut.run( @@ -572,7 +630,9 @@ create or replace package body ut is a_include_objects, a_exclude_objects, a_client_character_set, - a_force_manual_rollback + a_force_manual_rollback, + a_random_test_order, + a_random_test_order_seed ); end; diff --git a/source/api/ut.pks b/source/api/ut.pks index 25c9e6493..7d3b191b3 100644 --- a/source/api/ut.pks +++ b/source/api/ut.pks @@ -55,7 +55,9 @@ create or replace package ut authid current_user as a_test_file_mappings ut_file_mappings := null, a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, - a_client_character_set varchar2 := null + a_client_character_set varchar2 := null, + a_random_test_order integer := 0, + a_random_test_order_seed positive := null ) return ut_varchar2_rows pipelined; function run( @@ -66,7 +68,9 @@ create or replace package ut authid current_user as a_test_files ut_varchar2_list, a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, - a_client_character_set varchar2 := null + a_client_character_set varchar2 := null, + a_random_test_order integer := 0, + a_random_test_order_seed positive := null ) return ut_varchar2_rows pipelined; function run( @@ -78,7 +82,9 @@ create or replace package ut authid current_user as a_test_file_mappings ut_file_mappings := null, a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, - a_client_character_set varchar2 := null + a_client_character_set varchar2 := null, + a_random_test_order integer := 0, + a_random_test_order_seed positive := null ) return ut_varchar2_rows pipelined; function run( @@ -90,7 +96,9 @@ create or replace package ut authid current_user as a_test_files ut_varchar2_list, a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, - a_client_character_set varchar2 := null + a_client_character_set varchar2 := null, + a_random_test_order integer := 0, + a_random_test_order_seed positive := null ) return ut_varchar2_rows pipelined; function run( @@ -102,7 +110,9 @@ create or replace package ut authid current_user as a_test_file_mappings ut_file_mappings := null, a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, - a_client_character_set varchar2 := null + a_client_character_set varchar2 := null, + a_random_test_order integer := 0, + a_random_test_order_seed positive := null ) return ut_varchar2_rows pipelined; function run( @@ -114,7 +124,9 @@ create or replace package ut authid current_user as a_test_files ut_varchar2_list, a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, - a_client_character_set varchar2 := null + a_client_character_set varchar2 := null, + a_random_test_order integer := 0, + a_random_test_order_seed positive := null ) return ut_varchar2_rows pipelined; procedure run( @@ -126,7 +138,9 @@ create or replace package ut authid current_user as a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ); procedure run( @@ -138,7 +152,9 @@ create or replace package ut authid current_user as a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ); procedure run( @@ -151,7 +167,9 @@ create or replace package ut authid current_user as a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ); procedure run( @@ -164,7 +182,9 @@ create or replace package ut authid current_user as a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ); procedure run( @@ -177,7 +197,9 @@ create or replace package ut authid current_user as a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ); procedure run( @@ -190,7 +212,9 @@ create or replace package ut authid current_user as a_include_objects ut_varchar2_list := null, a_exclude_objects ut_varchar2_list := null, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ); /** diff --git a/source/api/ut_runner.pkb b/source/api/ut_runner.pkb index 10485c346..28ec1a610 100644 --- a/source/api/ut_runner.pkb +++ b/source/api/ut_runner.pkb @@ -88,13 +88,16 @@ create or replace package body ut_runner is a_exclude_objects ut_varchar2_list := null, a_fail_on_errors boolean := false, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ) is - l_run ut_run; - l_coverage_schema_names ut_varchar2_rows; - l_exclude_object_names ut_object_names := ut_object_names(); - l_include_object_names ut_object_names; - l_paths ut_varchar2_list := ut_varchar2_list(); + l_run ut_run; + l_coverage_schema_names ut_varchar2_rows; + l_exclude_object_names ut_object_names := ut_object_names(); + l_include_object_names ut_object_names; + l_paths ut_varchar2_list := ut_varchar2_list(); + l_random_test_order_seed positive; begin ut_event_manager.initialize(); if a_reporters is not empty then @@ -107,7 +110,12 @@ create or replace package body ut_runner is ut_event_manager.trigger_event(ut_event_manager.gc_initialize); ut_event_manager.trigger_event(ut_event_manager.gc_debug, ut_run_info()); - + if a_random_test_order_seed is not null then + l_random_test_order_seed := a_random_test_order_seed; + elsif a_random_test_order then + dbms_random.seed( to_char(systimestamp,'yyyyddmmhh24missffff') ); + l_random_test_order_seed := trunc(dbms_random.value(1, 1000000000)); + end if; if a_paths is null or a_paths is empty or a_paths.count = 1 and a_paths(1) is null then l_paths := ut_varchar2_list(sys_context('userenv', 'current_schema')); else @@ -144,10 +152,11 @@ create or replace package body ut_runner is l_include_object_names, set(a_source_file_mappings), set(a_test_file_mappings), - a_client_character_set + a_client_character_set, + l_random_test_order_seed ); - ut_suite_manager.configure_execution_by_path(l_paths, l_run.items); + ut_suite_manager.configure_execution_by_path(l_paths, l_run.items, l_random_test_order_seed); if a_force_manual_rollback then l_run.set_rollback_type( a_rollback_type => ut_utils.gc_rollback_manual, a_force => true ); end if; diff --git a/source/api/ut_runner.pks b/source/api/ut_runner.pks index e111933b5..ab2e1f6bb 100644 --- a/source/api/ut_runner.pks +++ b/source/api/ut_runner.pks @@ -68,7 +68,9 @@ create or replace package ut_runner authid current_user is a_exclude_objects ut_varchar2_list := null, a_fail_on_errors boolean := false, a_client_character_set varchar2 := null, - a_force_manual_rollback boolean := false + a_force_manual_rollback boolean := false, + a_random_test_order boolean := false, + a_random_test_order_seed positive := null ); /** diff --git a/source/core/annotations/ut_annotation_manager.pkb b/source/core/annotations/ut_annotation_manager.pkb index 6dbe1d3ec..87ad51475 100644 --- a/source/core/annotations/ut_annotation_manager.pkb +++ b/source/core/annotations/ut_annotation_manager.pkb @@ -249,5 +249,38 @@ create or replace package body ut_annotation_manager as ut_annotation_cache_manager.purge_cache(a_object_owner, a_object_type); end; + function hash_suite_path(a_path varchar2, a_random_seed positiven) return varchar2 is + l_start_pos pls_integer := 1; + l_end_pos pls_integer := 1; + l_result varchar2(4000); + l_item varchar2(4000); + l_at_end boolean := false; + begin + if a_random_seed is null then + l_result := a_path; + end if; + if a_path is not null then + loop + l_end_pos := instr(a_path,'.',l_start_pos); + if l_end_pos = 0 then + l_end_pos := length(a_path)+1; + l_at_end := true; + end if; + l_item := substr(a_path,l_start_pos,l_end_pos-l_start_pos); + if l_item is not null then + l_result := + l_result || + dbms_crypto.hash( + to_char( dbms_utility.get_hash_value( l_item, 1, a_random_seed ) ), + dbms_crypto.hash_sh1 + ); + end if; + exit when l_at_end; + l_result := l_result || chr(0); + l_start_pos := l_end_pos + 1; + end loop; + end if; + return l_result; + end; end ut_annotation_manager; / diff --git a/source/core/annotations/ut_annotation_manager.pks b/source/core/annotations/ut_annotation_manager.pks index 829142bd5..25de25d58 100644 --- a/source/core/annotations/ut_annotation_manager.pks +++ b/source/core/annotations/ut_annotation_manager.pks @@ -51,5 +51,8 @@ create or replace package ut_annotation_manager authid current_user as */ procedure purge_cache(a_object_owner varchar2, a_object_type varchar2); + + function hash_suite_path(a_path varchar2, a_random_seed positiven) return varchar2; + end ut_annotation_manager; / diff --git a/source/core/types/ut_run.tpb b/source/core/types/ut_run.tpb index 21a5ef36e..d5ace34a7 100644 --- a/source/core/types/ut_run.tpb +++ b/source/core/types/ut_run.tpb @@ -18,23 +18,22 @@ create or replace type body ut_run as constructor function ut_run( self in out nocopy ut_run, - a_items ut_suite_items, - a_run_paths ut_varchar2_list := null, - a_schema_names ut_varchar2_rows := null, - a_exclude_objects ut_object_names := null, - a_include_objects ut_object_names := null, - a_project_file_mappings ut_file_mappings := null, - a_test_file_mappings ut_file_mappings := null, - a_client_character_set varchar2 := null + a_items ut_suite_items, + a_run_paths ut_varchar2_list := null, + a_schema_names ut_varchar2_rows := null, + a_exclude_objects ut_object_names := null, + a_include_objects ut_object_names := null, + a_project_file_mappings ut_file_mappings := null, + a_test_file_mappings ut_file_mappings := null, + a_client_character_set varchar2 := null, + a_random_test_order_seed positive := null ) return self as result is - l_coverage_schema_names ut_varchar2_rows; - l_coverage_options ut_coverage_options; - l_exclude_objects ut_object_names; - begin + begin self.run_paths := a_run_paths; self.self_type := $$plsql_unit; self.items := a_items; self.client_character_set := lower(a_client_character_set); + self.random_test_order_seed := a_random_test_order_seed; self.results_count := ut_results_counter(); self.test_file_mappings := coalesce(a_test_file_mappings, ut_file_mappings()); self.coverage_options := ut_coverage_options( diff --git a/source/core/types/ut_run.tps b/source/core/types/ut_run.tps index e4309e126..5d5e65367 100644 --- a/source/core/types/ut_run.tps +++ b/source/core/types/ut_run.tps @@ -24,16 +24,18 @@ create or replace type ut_run under ut_suite_item ( coverage_options ut_coverage_options, test_file_mappings ut_file_mappings, client_character_set varchar2(100), + random_test_order_seed number(38,0), constructor function ut_run( self in out nocopy ut_run, - a_items ut_suite_items, - a_run_paths ut_varchar2_list := null, - a_schema_names ut_varchar2_rows := null, - a_exclude_objects ut_object_names := null, - a_include_objects ut_object_names := null, - a_project_file_mappings ut_file_mappings := null, - a_test_file_mappings ut_file_mappings := null, - a_client_character_set varchar2 := null + a_items ut_suite_items, + a_run_paths ut_varchar2_list := null, + a_schema_names ut_varchar2_rows := null, + a_exclude_objects ut_object_names := null, + a_include_objects ut_object_names := null, + a_project_file_mappings ut_file_mappings := null, + a_test_file_mappings ut_file_mappings := null, + a_client_character_set varchar2 := null, + a_random_test_order_seed positive := null ) return self as result, overriding member procedure mark_as_skipped(self in out nocopy ut_run), overriding member function do_execute(self in out nocopy ut_run) return boolean, diff --git a/source/core/ut_suite_manager.pkb b/source/core/ut_suite_manager.pkb index e2098caf9..51d3fa1e4 100644 --- a/source/core/ut_suite_manager.pkb +++ b/source/core/ut_suite_manager.pkb @@ -355,11 +355,12 @@ create or replace package body ut_suite_manager is end; function get_cached_suite_data( - a_object_owner varchar2, - a_path varchar2 := null, - a_object_name varchar2 := null, - a_procedure_name varchar2 := null, - a_skip_all_objects boolean := false + a_object_owner varchar2, + a_path varchar2 := null, + a_object_name varchar2 := null, + a_procedure_name varchar2 := null, + a_skip_all_objects boolean := false, + a_random_seed positive ) return t_cached_suites_cursor is l_path varchar2( 4000 ); l_result sys_refcursor; @@ -452,16 +453,25 @@ create or replace package body ut_suite_manager is ) select c.* from items c - order by c.object_owner, - replace(case - when c.self_type in ( 'UT_TEST' ) - then substr(c.path, 1, instr(c.path, '.', -1) ) - else c.path - end, '.', chr(0)) desc nulls last, + order by c.object_owner,]'|| + case + when a_random_seed is null then q'[ + replace( + case + when c.self_type in ( 'UT_TEST' ) + then substr(c.path, 1, instr(c.path, '.', -1) ) + else c.path + end, '.', chr(0) + ) desc nulls last, c.object_name desc, - c.line_no]' - using l_path, l_path, upper(a_object_name), upper(a_procedure_name); - + c.line_no, + :a_random_seed]' + else + l_ut_owner||'.ut_annotation_manager.hash_suite_path( + c.path, :a_random_seed + ) desc nulls last' + end + using l_path, l_path, upper(a_object_name), upper(a_procedure_name), a_random_seed; return l_result; end; @@ -547,7 +557,8 @@ create or replace package body ut_suite_manager is a_path varchar2 := null, a_object_name varchar2 := null, a_procedure_name varchar2 := null, - a_suites in out nocopy ut_suite_items + a_suites in out nocopy ut_suite_items, + a_random_seed positive ) is begin refresh_cache(a_owner_name); @@ -559,7 +570,8 @@ create or replace package body ut_suite_manager is a_path, a_object_name, a_procedure_name, - can_skip_all_objects_scan(a_owner_name) + can_skip_all_objects_scan(a_owner_name), + a_random_seed ) ); @@ -588,7 +600,8 @@ create or replace package body ut_suite_manager is a_path, a_object_name, a_procedure_name, - a_skip_all_objects + a_skip_all_objects, + null ) ); return l_suites; @@ -639,14 +652,18 @@ create or replace package body ut_suite_manager is return resolve_schema_names(l_paths); end; - function configure_execution_by_path(a_paths in ut_varchar2_list) return ut_suite_items is + function configure_execution_by_path(a_paths ut_varchar2_list, a_random_seed positive := null) return ut_suite_items is l_suites ut_suite_items := ut_suite_items(); begin configure_execution_by_path(a_paths, l_suites ); return l_suites; end; - procedure configure_execution_by_path(a_paths in ut_varchar2_list, a_suites out nocopy ut_suite_items) is + procedure configure_execution_by_path( + a_paths in ut_varchar2_list, + a_suites out nocopy ut_suite_items, + a_random_seed in positive := null + ) is l_paths ut_varchar2_list := a_paths; l_path_items t_path_items; l_path_item t_path_item; @@ -671,7 +688,8 @@ create or replace package body ut_suite_manager is l_path_item.suite_path, l_path_item.object_name, l_path_item.procedure_name, - a_suites + a_suites, + a_random_seed ); if a_suites.count = l_suites_count then if l_path_item.suite_path is not null then diff --git a/source/core/ut_suite_manager.pks b/source/core/ut_suite_manager.pks index 5fd82f9be..debfd4ac1 100644 --- a/source/core/ut_suite_manager.pks +++ b/source/core/ut_suite_manager.pks @@ -37,7 +37,7 @@ create or replace package ut_suite_manager authid current_user is * @return array containing root suites-ready to be executed * */ - function configure_execution_by_path(a_paths in ut_varchar2_list) return ut_suite_items; + function configure_execution_by_path(a_paths ut_varchar2_list, a_random_seed positive := null) return ut_suite_items; /** * Builds a hierarchical suites based on given suite-paths @@ -46,7 +46,11 @@ create or replace package ut_suite_manager authid current_user is * @param a_suites returned array containing root suites-ready to be executed * */ - procedure configure_execution_by_path(a_paths in ut_varchar2_list, a_suites out nocopy ut_suite_items); + procedure configure_execution_by_path( + a_paths in ut_varchar2_list, + a_suites out nocopy ut_suite_items, + a_random_seed in positive := null + ); /** * Cleanup paths by removing leading/trailing whitespace and making paths lowercase diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index abb07ee80..5ac8a4546 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -174,7 +174,7 @@ create or replace package body ut_compound_data_helper is function generate_equal_sql(a_col_name in varchar2) return varchar2 is begin - return ' a.'||a_col_name||q'[ = ]'||' e.'||a_col_name; + return ' decode(a.'||a_col_name||','||' e.'||a_col_name||',1,0) = 1 '; end; function generate_partition_stmt( diff --git a/source/reporters/ut_documentation_reporter.tpb b/source/reporters/ut_documentation_reporter.tpb index df6c25ab0..7cf9a02cc 100644 --- a/source/reporters/ut_documentation_reporter.tpb +++ b/source/reporters/ut_documentation_reporter.tpb @@ -209,6 +209,9 @@ create or replace type body ut_documentation_reporter is else self.print_green_text(l_summary_text); end if; + if a_run.random_test_order_seed is not null then + self.print_text('Tests were executed with random order seed '''||a_run.random_test_order_seed||'''.'); + end if; self.print_text(' '); (self as ut_reporter_base).after_calling_run(a_run); end; diff --git a/test/api/test_ut_run.pkb b/test/api/test_ut_run.pkb index 9e9fce0f7..fb18f18b8 100644 --- a/test/api/test_ut_run.pkb +++ b/test/api/test_ut_run.pkb @@ -1012,5 +1012,61 @@ Failures:% execute immediate 'drop package test_distributed_savepoint'; end; + procedure remove_time_from_results(a_results in out nocopy ut3.ut_varchar2_list) is + begin + for i in 1 .. a_results.count loop + a_results(i) := regexp_replace(a_results(i),'\[[0-9]*\.[0-9]+ sec\]',''); + a_results(i) := regexp_replace(a_results(i),'Finished in [0-9]*\.[0-9]+ seconds',''); + end loop; + end; + + procedure run_with_random_order is + l_random_results ut3.ut_varchar2_list; + l_results ut3.ut_varchar2_list; + begin + select * bulk collect into l_random_results + from table ( ut3.ut.run( 'ut3$user#.test_package_1', a_random_test_order_seed => 3 ) ) + where trim(column_value) is not null and column_value not like 'Finished in %' + and column_value not like '%Tests were executed with random order %'; + + select * bulk collect into l_results + from table ( ut3.ut.run( 'ut3$user#.test_package_1' ) ) + --TODO this condition should be removed once issues with unordered compare and 'blank text rows' are resolved. + where trim(column_value) is not null and column_value not like 'Finished in %'; + + remove_time_from_results(l_results); + remove_time_from_results(l_random_results); + + ut.expect(anydata.convertCollection(l_random_results)).to_equal(anydata.convertCollection(l_results)).unordered(); + ut.expect(anydata.convertCollection(l_random_results)).not_to_equal(anydata.convertCollection(l_results)); + end; + + procedure run_and_report_random_ord_seed is + l_actual ut3.ut_varchar2_list; + begin + select * bulk collect into l_actual + from table ( ut3.ut.run( 'ut3$user#.test_package_1', a_random_test_order_seed => 123456789 ) ); + + ut.expect( ut3.ut_utils.table_to_clob(l_actual) ).to_be_like( q'[%Tests were executed with random order seed '123456789'.%]' ); + end; + + procedure run_with_random_order_seed is + l_expected ut3.ut_varchar2_list; + l_actual ut3.ut_varchar2_list; + begin + + select * bulk collect into l_expected + from table ( ut3.ut.run( 'ut3$user#.test_package_1', a_random_test_order_seed => 3 ) ); + select * bulk collect into l_actual + from table ( ut3.ut.run( 'ut3$user#.test_package_1', a_random_test_order_seed => 3 ) ); + + remove_time_from_results(l_actual); + remove_time_from_results(l_expected); + l_actual.delete(l_actual.count); + l_expected.delete(l_expected.count); + + ut.expect(anydata.convertCollection(l_actual)).to_equal(anydata.convertCollection(l_expected)); + end; + end; / diff --git a/test/api/test_ut_run.pks b/test/api/test_ut_run.pks index 55d19e866..a2fae4b1f 100644 --- a/test/api/test_ut_run.pks +++ b/test/api/test_ut_run.pks @@ -144,5 +144,21 @@ create or replace package test_ut_run is procedure drop_bad_annot; --%endcontext + --%context(random_order) + --%displayname(Random test execution order) + --%beforeall(create_ut3$user#_tests) + --%afterall(drop_ut3$user#_tests) + + --%test(Runs tests in random order) + procedure run_with_random_order; + + --%test(Reports test random_test_order_seed) + procedure run_and_report_random_ord_seed; + + --%test(Runs tests in the same random order with provided seed) + procedure run_with_random_order_seed; + + --%endcontext + end; / diff --git a/test/core.pkb b/test/core.pkb index 60baa0190..b2405da9f 100644 --- a/test/core.pkb +++ b/test/core.pkb @@ -21,6 +21,7 @@ create or replace package body core is procedure global_setup is begin + dbms_output.enable(null); ut3.ut_coverage.set_develop_mode(true); --improve performance of test execution by disabling all compiler optimizations execute_autonomous('ALTER SESSION SET PLSQL_OPTIMIZE_LEVEL=0'); diff --git a/test/core/expectations/test_expectation_anydata.pkb b/test/core/expectations/test_expectation_anydata.pkb index 0b3a93261..9aa968b09 100644 --- a/test/core/expectations/test_expectation_anydata.pkb +++ b/test/core/expectations/test_expectation_anydata.pkb @@ -970,5 +970,29 @@ Rows: [ 60 differences, showing first 20 ] ut.expect(expectations.failed_expectations_data()).to_be_empty(); end; + procedure arr_empty_eq_arr_empty_unord is + begin + --Arrange + g_test_expected := anydata.convertCollection(t_tab_varchar(null)); + g_test_actual := anydata.convertCollection(t_tab_varchar(null)); + + --Act + ut3.ut.expect( g_test_actual ).to_equal( g_test_expected ).unordered(); + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + + end; + + procedure arr_empty_nqua_arr_e_unord is + begin + --Arrange + g_test_expected := anydata.convertCollection(t_tab_varchar('t')); + g_test_actual := anydata.convertCollection(t_tab_varchar(' ')); + + --Act + ut3.ut.expect( g_test_actual ).not_to_equal( g_test_expected ).unordered(); + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + + end; + end; / \ No newline at end of file diff --git a/test/core/expectations/test_expectation_anydata.pks b/test/core/expectations/test_expectation_anydata.pks index b9fbda93b..bd0ac95db 100644 --- a/test/core/expectations/test_expectation_anydata.pks +++ b/test/core/expectations/test_expectation_anydata.pks @@ -198,5 +198,10 @@ create or replace package test_expectation_anydata is --%test( Success when anydata object contains data from another anydata) procedure object_to_contain; + --%test ( Empty Array equal empty array ) + procedure arr_empty_eq_arr_empty_unord; + + --%test ( Empty Array not equal array with space ) + procedure arr_empty_nqua_arr_e_unord; end; /