diff --git a/examples/annoation_parser_testing/annotation_parser_test.sql b/examples/annoation_parser_testing/annotation_parser_test.sql new file mode 100644 index 000000000..d1ff43fcd --- /dev/null +++ b/examples/annoation_parser_testing/annotation_parser_test.sql @@ -0,0 +1,298 @@ +-- Annotation Manager Volume Benchmark +-- 1000 generated packages, cold cache per iteration + +set serveroutput on size unlimited +set timing off +set feedback off + +declare + c_packages constant pls_integer := 1000; + c_procs_per_pkg constant pls_integer := 20; + c_contexts_per_pkg constant pls_integer := 4; + c_pkg_prefix constant varchar2(20) := 'TST_GEN_PKG_'; + c_iterations constant pls_integer := 4; + c_perf_threshold constant number := 0.80; + + l_baseline_total_ms number := 0; + l_new_total_ms number := 0; + l_start timestamp; + l_elapsed interval day to second; + + -- Source holders + l_source_lines dbms_preprocessor.source_lines_t; + l_source_clob clob; + l_result ut_annotations; + + -- Correctness tracking + l_missing_from_cache pls_integer := 0; + l_count_mismatches pls_integer := 0; + l_mismatch_detail varchar2(32767); + + -- Cache check + l_cursor sys_refcursor; + l_annotated_obj ut_annotated_object; + l_name varchar2(128); + l_annotations ut_annotations; + l_cache_count pls_integer := 0; + + -- Per-package timing + l_pkg_baseline_ms number; + l_pkg_new_ms number; + l_pkg varchar2(30); + l_ratio number; + + -- Expected annotation count — mirrors generator logic exactly + function expected_annotation_count( + a_pkg_number pls_integer, + a_procs pls_integer, + a_contexts pls_integer + ) return pls_integer is + l_count pls_integer := 0; + begin + l_count := l_count + 2; -- %suite + %suitepath + l_count := l_count + 2; -- %beforeall + %afterall + l_count := l_count + 2; -- %beforeeach + %aftereach + + -- package level conditional + if mod(a_pkg_number, 3) = 0 then l_count := l_count + 1; end if; -- %displayname + if mod(a_pkg_number, 5) = 0 then l_count := l_count + 1; end if; -- %rollback + + -- contexts: each contributes %context + %endcontext + l_count := l_count + (a_contexts * 2); + + -- per procedure + for i in 1 .. a_procs loop + l_count := l_count + 1; -- %test + if mod(i, 3) = 0 then l_count := l_count + 1; end if; -- %tags + if mod(i, 6) = 0 then l_count := l_count + 1; end if; -- %throws + if mod(i, 8) = 0 then l_count := l_count + 1; end if; -- %disabled + if mod(i, 5) = 0 then l_count := l_count + 2; end if; -- %beforetest + %aftertest + end loop; + + return l_count; + end expected_annotation_count; + +begin + dbms_output.put_line('Packages : ' || c_packages); + dbms_output.put_line('Procs/pkg : ' || c_procs_per_pkg); + dbms_output.put_line('Contexts/pkg: ' || c_contexts_per_pkg); + dbms_output.put_line('Iterations : ' || c_iterations); + dbms_output.put_line('Started : ' || to_char(systimestamp,'YYYY-MM-DD HH24:MI:SS.FF3')); + + -- Phase 1: Performance benchmark (cold cache per package per iteration) + dbms_output.put_line(chr(10) || '-- Phase 1: Performance --'); + dbms_output.put_line( + rpad('package', 20) || ' | ' || + lpad('baseline_ms', 11) || ' | ' || + lpad('new_ms', 6) || ' | ' || + lpad('ratio', 5) || ' | ' || + 'status' + ); + dbms_output.put_line(rpad('-', 70, '-')); + + for p in 1 .. c_packages loop + l_pkg := c_pkg_prefix || lpad(p, 4, '0'); + + -- load source once per package + l_source_lines := dbms_preprocessor.get_post_processed_source( + object_type => 'PACKAGE', + schema_name => user, + object_name => l_pkg + ); + + dbms_lob.createtemporary(l_source_clob, true); + for i in 1 .. l_source_lines.count loop + dbms_lob.writeappend(l_source_clob, length(l_source_lines(i)), l_source_lines(i)); + end loop; + + -- baseline: clob version + l_pkg_baseline_ms := 0; + for iter in 1 .. c_iterations loop + ut_annotation_manager.purge_cache(user, 'PACKAGE'); + l_start := systimestamp; + l_result := ut_annotation_parser.parse_object_annotations(l_source_clob); + l_elapsed := systimestamp - l_start; + l_pkg_baseline_ms := l_pkg_baseline_ms + + extract(second from l_elapsed) * 1000; + end loop; + l_pkg_baseline_ms := l_pkg_baseline_ms / c_iterations; + + -- new: source_lines_t version + l_pkg_new_ms := 0; + for iter in 1 .. c_iterations loop + ut_annotation_manager.purge_cache(user, 'PACKAGE'); + l_start := systimestamp; + l_result := ut_annotation_parser.parse_object_annotations(l_source_lines); + l_elapsed := systimestamp - l_start; + l_pkg_new_ms := l_pkg_new_ms + + extract(second from l_elapsed) * 1000; + end loop; + l_pkg_new_ms := l_pkg_new_ms / c_iterations; + + -- accumulate + l_baseline_total_ms := l_baseline_total_ms + l_pkg_baseline_ms; + l_new_total_ms := l_new_total_ms + l_pkg_new_ms; + + l_ratio := round(l_pkg_new_ms / nullif(l_pkg_baseline_ms, 0), 3); + + -- print every 100 packages to avoid flooding output + if mod(p, 100) = 0 or p = 1 then + dbms_output.put_line( + rpad(l_pkg, 20) || ' | ' || + lpad(round(l_pkg_baseline_ms, 3), 11) || ' | ' || + lpad(round(l_pkg_new_ms, 3), 6) || ' | ' || + lpad(l_ratio, 5) || ' | ' || + case when l_ratio <= c_perf_threshold + then 'PASS (' || round((1-l_ratio)*100,1) || '% faster)' + else 'SLOW (ratio=' || l_ratio || ')' + end + ); + end if; + + dbms_lob.freetemporary(l_source_clob); + end loop; + + -- ------------------------------------------------------------------------- + -- Phase 1 summary + -- ------------------------------------------------------------------------- + l_ratio := round(l_new_total_ms / nullif(l_baseline_total_ms, 0), 3); + + dbms_output.put_line(rpad('-', 70, '-')); + dbms_output.put_line('Baseline total ms : ' || round(l_baseline_total_ms, 3)); + dbms_output.put_line('New total ms : ' || round(l_new_total_ms, 3)); + dbms_output.put_line('Overall ratio : ' || l_ratio); + dbms_output.put_line('Improvement : ' || round((1 - l_ratio) * 100, 1) || '%'); + + -- Phase 2: Cache presence check + dbms_output.put_line(chr(10) || '-- Phase 2: Cache presence --'); + + ut_annotation_manager.purge_cache(user, 'PACKAGE'); + ut_annotation_manager.rebuild_annotation_cache(user, 'PACKAGE'); + + -- query cache tables directly to avoid validate_annotation_cache side effects + open l_cursor for + select ut_annotated_object( + i.object_owner, i.object_name, i.object_type, i.parse_time, + cast( + collect( + ut_annotation( + c.annotation_position, c.annotation_name, + c.annotation_text, c.subobject_name + ) order by c.annotation_position + ) as ut_annotations + ) + ) + from ut_annotation_cache_info i + join ut_annotation_cache c on i.cache_id = c.cache_id + where i.object_owner = user + and i.object_type = 'PACKAGE' + and i.object_name like c_pkg_prefix || '%' + group by i.object_owner, i.object_type, i.object_name, i.parse_time; + + loop + fetch l_cursor into l_annotated_obj; + exit when l_cursor%notfound; + l_cache_count := l_cache_count + 1; + end loop; + close l_cursor; + + dbms_output.put_line('Expected in cache : ' || c_packages); + dbms_output.put_line('Found in cache : ' || l_cache_count); + dbms_output.put_line('Cache check : ' || + case when l_cache_count = c_packages + then 'PASS — all ' || c_packages || ' packages found in cache' + else 'FAIL — missing ' || (c_packages - l_cache_count) || ' packages from cache' + end + ); + + -- Phase 3: Annotation count correctness + dbms_output.put_line(chr(10) || '-- Phase 3: Annotation count correctness --'); + + -- reopen same direct cache query for correctness check + open l_cursor for + select ut_annotated_object( + i.object_owner, i.object_name, i.object_type, i.parse_time, + cast( + collect( + ut_annotation( + c.annotation_position, c.annotation_name, + c.annotation_text, c.subobject_name + ) order by c.annotation_position + ) as ut_annotations + ) + ) + from ut_annotation_cache_info i + join ut_annotation_cache c on i.cache_id = c.cache_id + where i.object_owner = user + and i.object_type = 'PACKAGE' + and i.object_name like c_pkg_prefix || '%' + group by i.object_owner, i.object_type, i.object_name, i.parse_time; + + loop + fetch l_cursor into l_annotated_obj; + exit when l_cursor%notfound; + l_name := l_annotated_obj.object_name; + l_annotations := l_annotated_obj.annotations; + + declare + l_pkg_number pls_integer; + l_expected pls_integer; + begin + l_pkg_number := to_number(substr(l_name, length(c_pkg_prefix) + 1)); + l_expected := expected_annotation_count( + l_pkg_number, + c_procs_per_pkg, + c_contexts_per_pkg + ); + + if l_annotations.count != l_expected then + l_count_mismatches := l_count_mismatches + 1; + if l_count_mismatches <= 10 then + l_mismatch_detail := l_mismatch_detail || chr(10) || + ' ' || l_name || + ': expected=' || l_expected || + ' actual=' || l_annotations.count; + end if; + end if; + end; + end loop; + close l_cursor; + + dbms_output.put_line('Packages checked : ' || c_packages); + dbms_output.put_line('Count mismatches : ' || l_count_mismatches); + if l_count_mismatches > 0 then + dbms_output.put_line('Mismatch detail (first 10):' || l_mismatch_detail); + end if; + dbms_output.put_line('Count check : ' || + case when l_count_mismatches = 0 + then 'PASS — all annotation counts match expected' + else 'FAIL — ' || l_count_mismatches || ' packages have wrong annotation count' + end + ); + + dbms_output.put_line(chr(10) || '================================================='); + dbms_output.put_line('FINAL RESULTS'); + dbms_output.put_line('================================================='); + dbms_output.put_line('Performance : ' || + case when l_ratio <= c_perf_threshold then 'PASS' else 'FAIL' end); + dbms_output.put_line('Cache : ' || + case when l_cache_count = c_packages then 'PASS' else 'FAIL' end); + dbms_output.put_line('Counts : ' || + case when l_count_mismatches = 0 then 'PASS' else 'FAIL' end); + dbms_output.put_line('Overall : ' || + case when l_ratio <= c_perf_threshold + and l_cache_count = c_packages + and l_count_mismatches = 0 + then 'PASS' + else 'FAIL' + end + ); + dbms_output.put_line('Finished : ' || + to_char(systimestamp, 'YYYY-MM-DD HH24:MI:SS.FF3')); + dbms_output.put_line('================================================='); + + -- cleanup + ut_annotation_manager.purge_cache(user, 'PACKAGE'); + +end; +/ diff --git a/examples/annoation_parser_testing/create_test_packages.sql b/examples/annoation_parser_testing/create_test_packages.sql new file mode 100644 index 000000000..94a7f3de2 --- /dev/null +++ b/examples/annoation_parser_testing/create_test_packages.sql @@ -0,0 +1,185 @@ +declare + c_packages constant pls_integer := 1000; + c_procs_per_pkg constant pls_integer := 20; + + l_spec varchar2(32767); + l_body varchar2(32767); + l_pkg varchar2(30); + l_errors pls_integer := 0; + + procedure append_to_source(a_source in out nocopy varchar2, a_text varchar2) is + begin + a_source := a_source || a_text; + end; + + procedure append_line(a_source in out nocopy varchar2, a_text varchar2 default null) is + begin + a_source := a_source || a_text || chr(10); + end; + +begin + + for r in ( + select object_name + from user_objects + where object_name like 'TST_GEN_PKG_%' + and object_type = 'PACKAGE' + order by object_name + ) loop + begin + execute immediate 'drop package ' || r.object_name; + exception + when others then + dbms_output.put_line('ERROR dropping ' || r.object_name || ': ' || sqlerrm); + end; + end loop; + dbms_output.put_line('Existing packages dropped.'); + + for p in 1 .. c_packages loop + l_pkg := 'TST_GEN_PKG_' || lpad(p, 4, '0'); + l_spec := null; + l_body := null; + + -- Package spec + append_line(l_spec, 'create or replace package ' || l_pkg || ' as'); + append_line(l_spec); + append_line(l_spec, ' --%suite(Generated suite ' || p || ')'); + append_line(l_spec, ' --%suitepath(generated.vol_test)'); + + if mod(p, 3) = 0 then + append_line(l_spec, ' --%displayname(Suite display name for pkg ' || p || ')'); + end if; + if mod(p, 5) = 0 then + append_line(l_spec, ' --%rollback(manual)'); + end if; + + append_line(l_spec); + append_line(l_spec, ' --%beforeall'); + append_line(l_spec, ' procedure setup_suite;'); + append_line(l_spec); + append_line(l_spec, ' --%afterall'); + append_line(l_spec, ' procedure teardown_suite;'); + append_line(l_spec); + append_line(l_spec, ' --%beforeeach'); + append_line(l_spec, ' procedure setup_test;'); + append_line(l_spec); + append_line(l_spec, ' --%aftereach'); + append_line(l_spec, ' procedure teardown_test;'); + append_line(l_spec); + + for i in 1 .. c_procs_per_pkg loop + declare + l_proc varchar2(30) := 'test_proc_' || lpad(i, 3, '0'); + begin + if mod(i - 1, 5) = 0 then + append_line(l_spec, ' --%context(context_' || ceil(i/5) || ')'); + append_line(l_spec); + end if; + + if mod(i, 4) = 0 then + append_line(l_spec, ' /* multi-line comment for ' || l_proc || ' */'); + end if; + + append_line(l_spec, ' --%test(Test ' || i || ' in pkg ' || p || ')'); + + if mod(i, 3) = 0 then + append_line(l_spec, ' --%tags(tag_' || mod(i,5) || ',smoke)'); + end if; + if mod(i, 6) = 0 then + append_line(l_spec, ' --%throws(-20001)'); + end if; + if mod(i, 8) = 0 then + append_line(l_spec, ' --%disabled(generated disabled test ' || i || ')'); + end if; + if mod(i, 5) = 0 then + append_line(l_spec, ' --%beforetest(setup_test)'); + append_line(l_spec, ' --%aftertest(teardown_test)'); + end if; + + append_line(l_spec, ' -- regular comment'); + append_line(l_spec, ' procedure ' || l_proc || ';'); + append_line(l_spec); + + if mod(i, 5) = 0 then + append_line(l_spec, ' --%endcontext'); + append_line(l_spec); + end if; + end; + end loop; + + append_line(l_spec, 'end ' || l_pkg || ';'); + + -- ----------------------------------------------------------------------- + -- Package body + -- ----------------------------------------------------------------------- + append_line(l_body, 'create or replace package body ' || l_pkg || ' as'); + append_line(l_body); + + for stub in ( + select column_value as proc_name + from table(ut_varchar2_list( + 'setup_suite','teardown_suite', + 'setup_test','teardown_test' + )) + ) loop + append_line(l_body, ' procedure ' || stub.proc_name || ' is'); + append_line(l_body, ' begin'); + append_line(l_body, ' null;'); + append_line(l_body, ' end ' || stub.proc_name || ';'); + append_line(l_body); + end loop; + + for i in 1 .. c_procs_per_pkg loop + declare + l_proc varchar2(30) := 'test_proc_' || lpad(i, 3, '0'); + begin + append_line(l_body, ' procedure ' || l_proc || ' is'); + append_line(l_body, ' begin'); + if mod(i, 4) = 0 then + append_line(l_body, ' ut.expect(''/* not a comment */'').to_equal(''/* not a comment */'');'); + elsif mod(i, 4) = 1 then + append_line(l_body, ' ut.expect(' || i || ').to_be_greater_than(0);'); + elsif mod(i, 4) = 2 then + append_line(l_body, ' ut.expect(q''[-- not a comment]'').to_equal(q''[-- not a comment]'');'); + else + append_line(l_body, ' ut.expect(''test_' || i || ''').not_to_be_null();'); + end if; + append_line(l_body, ' end ' || l_proc || ';'); + append_line(l_body); + end; + end loop; + + append_line(l_body, 'end ' || l_pkg || ';'); + + -- Execute DDL — with error capture so one bad package doesn't stop all + begin + execute immediate l_spec; + execute immediate l_body; + exception + when others then + l_errors := l_errors + 1; + dbms_output.put_line('ERROR on ' || l_pkg || ': ' || sqlerrm); + end; + + if mod(p, 100) = 0 then + dbms_output.put_line('Created ' || p || ' packages...'); + end if; + + end loop; + + dbms_output.put_line('----------------------------------------'); + dbms_output.put_line('Done. packages=' || c_packages || + ' errors=' || l_errors); + + -- quick sanity check + declare + l_count pls_integer; + begin + select count(*) into l_count + from user_objects + where object_name like 'TST_GEN_PKG_%' + and object_type = 'PACKAGE'; + dbms_output.put_line('Packages in user_objects: ' || l_count); + end; +end; +/ diff --git a/source/core/annotations/ut_annotation_manager.pkb b/source/core/annotations/ut_annotation_manager.pkb index 65f7b3e40..6767411f1 100644 --- a/source/core/annotations/ut_annotation_manager.pkb +++ b/source/core/annotations/ut_annotation_manager.pkb @@ -143,34 +143,37 @@ create or replace package body ut_annotation_manager as l_parse_time date := sysdate; pragma autonomous_transaction; begin - loop - fetch a_sources_cursor bulk collect into l_names, l_lines limit c_lines_fetch_limit; - for i in 1 .. l_names.count loop - if l_names(i) != l_name then - l_annotations := ut_annotation_parser.parse_object_annotations(l_object_lines, a_object_type); - ut_annotation_cache_manager.update_cache( - ut_annotated_object(a_object_owner, l_name, a_object_type, l_parse_time, l_annotations) - ); - l_object_lines.delete; - end if; - - l_name := l_names(i); - l_object_lines(l_object_lines.count+1) := l_lines(i); - end loop; - exit when a_sources_cursor%notfound; + begin + loop + fetch a_sources_cursor bulk collect into l_names, l_lines limit c_lines_fetch_limit; + for i in 1 .. l_names.count loop + if l_names(i) != l_name then + l_annotations := ut_annotation_parser.parse_object_annotations(l_object_lines, a_object_type); + ut_annotation_cache_manager.update_cache( + ut_annotated_object(a_object_owner, l_name, a_object_type, l_parse_time, l_annotations) + ); + l_object_lines.delete; + end if; + + l_name := l_names(i); + l_object_lines(l_object_lines.count+1) := l_lines(i); + end loop; + exit when a_sources_cursor%notfound; - end loop; - if a_sources_cursor%rowcount > 0 then - l_annotations := ut_annotation_parser.parse_object_annotations(l_object_lines, a_object_type); - ut_annotation_cache_manager.update_cache( - ut_annotated_object(a_object_owner, l_name, a_object_type, l_parse_time, l_annotations) - ); - l_object_lines.delete; + end loop; + if a_sources_cursor%rowcount > 0 then + l_annotations := ut_annotation_parser.parse_object_annotations(l_object_lines, a_object_type); + ut_annotation_cache_manager.update_cache( + ut_annotated_object(a_object_owner, l_name, a_object_type, l_parse_time, l_annotations) + ); + l_object_lines.delete; + end if; + end; + if a_sources_cursor%isopen then + close a_sources_cursor; end if; - close a_sources_cursor; end; - procedure validate_annotation_cache( a_object_owner varchar2, a_object_type varchar2, @@ -231,23 +234,24 @@ create or replace package body ut_annotation_manager as function get_source_from_sql_text(a_object_name varchar2, a_sql_text ora_name_list_t, a_parts binary_integer) return sys_refcursor is l_sql_clob clob; - l_sql_lines ut_varchar2_rows := ut_varchar2_rows(); + l_sql_lines dbms_preprocessor.source_lines_t := dbms_preprocessor.source_lines_t(); + l_sql_lines_clob ut_varchar2_list := ut_varchar2_list(); l_result sys_refcursor; begin if a_parts > 0 then for i in 1..a_parts loop ut_utils.append_to_clob(l_sql_clob, a_sql_text(i)); end loop; - l_sql_clob := ut_utils.replace_multiline_comments(l_sql_clob); - -- replace comment lines that contain "-- create or replace" - l_sql_clob := regexp_replace(l_sql_clob, '^.*[-]{2,}\s*create(\s+or\s+replace).*$', modifier => 'mi'); - -- remove the "create [or replace] [[non]editionable] " so that we have only "type|package" for parsing - -- needed for dbms_preprocessor - l_sql_clob := regexp_replace(l_sql_clob, '^(.*?\s*create(\s+or\s+replace)?(\s+(editionable|noneditionable))?\s+?)((package|type).*)', '\5', 1, 1, 'ni'); - -- remove "OWNER." from create or replace statement. - -- Owner is not supported along with AUTHID - see issue https://github.com/utPLSQL/utPLSQL/issues/1088 - l_sql_clob := regexp_replace(l_sql_clob, '^(package|type)\s+("?[[:alpha:]][[:alnum:]$#_]*"?\.)(.*)', '\1 \3', 1, 1, 'ni'); - l_sql_lines := ut_utils.convert_collection( ut_utils.clob_to_table(l_sql_clob) ); + + l_sql_lines_clob := ut_utils.clob_to_table(l_sql_clob); + for i in 1..l_sql_lines_clob.count loop + l_sql_lines(i) := l_sql_lines_clob(i); + end loop; + + -- replace multiline comments that contain "-- create or replace" with single line comment to avoid parsing issues with dbms_preprocessor + l_sql_lines := ut_utils.replace_multiline_comments(l_sql_lines); + -- strip CREATE header (possibly split across lines) while preserving line numbers + l_sql_lines := ut_utils.strip_create_header_lines(l_sql_lines); end if; open l_result for select /*+ no_parallel */ a_object_name as name, column_value||chr(10) as text from table(l_sql_lines); diff --git a/source/core/annotations/ut_annotation_parser.pkb b/source/core/annotations/ut_annotation_parser.pkb index 10bb76b3c..d356a1ebf 100644 --- a/source/core/annotations/ut_annotation_parser.pkb +++ b/source/core/annotations/ut_annotation_parser.pkb @@ -22,12 +22,10 @@ create or replace package body ut_annotation_parser as type tt_comment_list is table of varchar2(32767) index by binary_integer; gc_annotation_qualifier constant varchar2(1) := '%'; - gc_annot_comment_pattern constant varchar2(30) := '^( |'||chr(09)||')*-- *('||gc_annotation_qualifier||'.*?)$'; -- chr(09) is a tab character gc_comment_replacer_patter constant varchar2(50) := '{COMMENT#%N%}'; gc_comment_replacer_regex_ptrn constant varchar2(25) := '{COMMENT#(\d+)}'; gc_regexp_identifier constant varchar2(50) := '[[:alpha:]][[:alnum:]$#_]*'; - gc_annotation_block_pattern constant varchar2(200) := '(({COMMENT#.+}'||chr(10)||')+)( |'||chr(09)||')*(procedure|function)\s+(' || - gc_regexp_identifier || ')'; + gc_annotation_pattern constant varchar2(50) := gc_annotation_qualifier || gc_regexp_identifier || '[ '||chr(9)||']*(\(.*?\)\s*?$)?'; @@ -72,133 +70,144 @@ create or replace package body ut_annotation_parser as a_subobject_name varchar2 := null ) is l_loop_index binary_integer := 1; - l_annotation_index binary_integer; + l_annotation_index binary_integer := 1; begin - -- loop while there are unprocessed comment blocks - while 0 != nvl(regexp_instr(srcstr => a_source - ,pattern => gc_comment_replacer_regex_ptrn - ,occurrence => l_loop_index - ,subexpression => 1) - ,0) loop + -- loop while there are unprocessed comment blocks + while l_annotation_index is not null loop -- define index of the comment block and get it's content from cache l_annotation_index := regexp_substr( a_source ,gc_comment_replacer_regex_ptrn ,1 ,l_loop_index ,subexpression => 1); - add_annotation( a_annotations, l_annotation_index, a_comments( l_annotation_index ), a_subobject_name ); - l_loop_index := l_loop_index + 1; + if l_annotation_index is not null then + add_annotation( a_annotations, l_annotation_index, a_comments( l_annotation_index ), a_subobject_name ); + l_loop_index := l_loop_index + 1; + end if; end loop; end add_annotations; - procedure add_procedure_annotations(a_annotations in out nocopy ut_annotations, a_source clob, a_comments in out nocopy tt_comment_list) is - l_proc_comments varchar2(32767); - l_proc_name varchar2(250); - l_annot_proc_ind number; - l_annot_proc_block varchar2(32767); + procedure add_procedure_annotations( + a_annotations in out nocopy ut_annotations, + a_source in dbms_preprocessor.source_lines_t, + a_comments in out nocopy tt_comment_list + ) is + l_proc_comments varchar2(32767); + l_proc_name varchar2(250); + l_line varchar2(32767); + l_in_comment_block boolean := false; + l_i binary_integer; begin - -- loop through procedures and functions of the package and get all the comment blocks just before it's declaration - l_annot_proc_ind := 1; - loop - --find annotated procedure index - l_annot_proc_ind := regexp_instr(srcstr => a_source - ,pattern => gc_annotation_block_pattern - ,occurrence => 1 - ,modifier => 'i' - ,position => l_annot_proc_ind); - exit when l_annot_proc_ind = 0; - - --get the annotations with procedure name - l_annot_proc_block := regexp_substr(srcstr => a_source - ,pattern => gc_annotation_block_pattern - ,position => l_annot_proc_ind - ,occurrence => 1 - ,modifier => 'i'); - - --extract the annotations - l_proc_comments := trim(regexp_substr(srcstr => l_annot_proc_block - ,pattern => gc_annotation_block_pattern - ,modifier => 'i' - ,subexpression => 1)); - --extract the procedure name - l_proc_name := trim(regexp_substr(srcstr => l_annot_proc_block - ,pattern => gc_annotation_block_pattern - ,modifier => 'i' - ,subexpression => 5)); - - -- parse the comment block for the syntactically correct annotations and store them as an array - add_annotations(a_annotations, l_proc_comments, a_comments, l_proc_name); - - l_annot_proc_ind := instr(a_source, ';', l_annot_proc_ind + length(l_annot_proc_block) ); + l_i := 1; + while l_i <= a_source.count loop + l_line := a_source(l_i); + -- Comment placeholder line: start/continue accumulating a block + if instr(l_line, chr(123) ||'COMMENT#') > 0 then + if l_in_comment_block then + l_proc_comments := l_proc_comments || l_line || chr(10); + else + l_in_comment_block := true; + l_proc_comments := l_line || chr(10); + end if; + l_i := l_i + 1; + -- Whitespace-only line: allowed between comment block and proc decl + elsif l_in_comment_block and trim(replace(l_line, chr(9))) is null then + l_i := l_i + 1; + -- procedure/function declaration following a comment block + elsif l_in_comment_block + and regexp_like(l_line, '^\s*(procedure|function)\s+', 'i') + then + -- extract just the identifier name (subexpression 2) + l_proc_name := trim(regexp_substr(srcstr =>l_line + ,pattern => '^\s*(procedure|function)\s+('||gc_regexp_identifier||')' + ,modifier => 'i' + ,subexpression => 2)); + + -- pass accumulated comment placeholders + proc name to add_annotations + add_annotations(a_annotations, l_proc_comments, a_comments, l_proc_name); + + -- reset comment block state + l_in_comment_block := false; + l_proc_comments := null; + + -- advance past proc header to the terminating ';' + -- the header may span multiple lines e.g. with multi-line parameter lists + while l_i <= a_source.count loop + exit when instr(a_source(l_i), ';') > 0; + l_i := l_i + 1; + end loop; + l_i := l_i + 1; -- step past the ';' line itself + -- Any other line: reset comment block accumulator + else + l_in_comment_block := false; + l_proc_comments := null; + l_i := l_i + 1; + end if; end loop; end add_procedure_annotations; - function extract_and_replace_comments(a_source in out nocopy clob) return tt_comment_list is - l_comments tt_comment_list; - l_comment_pos binary_integer; - l_comment_line binary_integer; + function extract_and_replace_comments( + a_source in out nocopy dbms_preprocessor.source_lines_t + ) return tt_comment_list is + l_comments tt_comment_list; + l_line varchar2(32767); + l_comment_pos binary_integer; l_comment_replacer varchar2(50); - l_source clob := a_source; begin - l_comment_pos := 1; - loop - - l_comment_pos := regexp_instr(srcstr => a_source - ,pattern => gc_annot_comment_pattern - ,occurrence => 1 - ,modifier => 'm' - ,position => l_comment_pos); - - exit when l_comment_pos = 0; - - -- position index is shifted by 1 because gc_annot_comment_pattern contains ^ as first sign - -- but after instr index already points to the char on that line - l_comment_pos := l_comment_pos-1; - l_comment_line := length(substr(a_source,1,l_comment_pos))-length(replace(substr(a_source,1,l_comment_pos),chr(10)))+1; - l_comments(l_comment_line) := trim(regexp_substr(srcstr => a_source - ,pattern => gc_annot_comment_pattern - ,occurrence => 1 - ,position => l_comment_pos - ,modifier => 'm' - ,subexpression => 2)); - - l_comment_replacer := replace(gc_comment_replacer_patter, '%N%', l_comment_line); - - l_source := regexp_replace(srcstr => a_source - ,pattern => gc_annot_comment_pattern - ,replacestr => l_comment_replacer - ,position => l_comment_pos - ,occurrence => 1 - ,modifier => 'm'); - dbms_lob.freetemporary(a_source); - a_source := l_source; - dbms_lob.freetemporary(l_source); - l_comment_pos := l_comment_pos + length(l_comment_replacer); + for i in 1 .. a_source.count loop + l_line := a_source(i); + + -- fast path: skip lines that can't possibly match + -- must contain '--' and '%' to be an annotation comment + if instr(l_line, '--') = 0 or instr(l_line, gc_annotation_qualifier) = 0 then + continue; + end if; + + -- find '--' on the line + l_comment_pos := instr(l_line, '--'); + + -- verify everything before '--' is only spaces/tabs (matches ^ *( |\t)*--) + if trim(replace(substr(l_line, 1, l_comment_pos - 1), chr(9))) is not null then + continue; + end if; + + -- skip '--' and any spaces after it, then check for annotation qualifier '%' + l_comment_pos := l_comment_pos + 2; + -- skip optional spaces between -- and % + while l_comment_pos <= length(l_line) + and substr(l_line, l_comment_pos, 1) = ' ' + loop + l_comment_pos := l_comment_pos + 1; + end loop; + + -- must start with annotation qualifier at this position + if substr(l_line, l_comment_pos, 1) != gc_annotation_qualifier then + continue; + end if; + + -- extract annotation text (from % to end of line, trimmed) + l_comments(i) := trim(substr(l_line, l_comment_pos)); + + -- replace line with placeholder, preserving line number in token + l_comment_replacer := replace(gc_comment_replacer_patter, '%N%', i); + a_source(i) := l_comment_replacer; end loop; - ut_utils.debug_log(a_source); return l_comments; end extract_and_replace_comments; ------------------------------------------------------------ --public definitions ------------------------------------------------------------ - - function parse_object_annotations(a_source clob) return ut_annotations is - l_source clob := a_source; + function parse_object_annotations(a_source dbms_preprocessor.source_lines_t) return ut_annotations is + l_source dbms_preprocessor.source_lines_t := a_source; l_comments tt_comment_list; l_annotations ut_annotations := ut_annotations(); l_result ut_annotations; l_comment_index positive; begin - - l_source := ut_utils.replace_multiline_comments(l_source); - - -- replace all single line comments with {COMMENT#12} element and store it's content for easier processing - -- this call modifies l_source + l_source := ut_utils.replace_multiline_comments(l_source); l_comments := extract_and_replace_comments(l_source); - add_procedure_annotations(l_annotations, l_source, l_comments); - delete_processed_comments(l_comments, l_annotations); --at this point, only the comments not related to procedures are left, so we process them all as top-level @@ -208,16 +217,15 @@ create or replace package body ut_annotation_parser as l_comment_index := l_comments.next(l_comment_index); end loop; - dbms_lob.freetemporary(l_source); - - select /*+ no_parallel */ value(x) bulk collect into l_result from table(l_annotations) x order by x.position; + select /*+ no_parallel */ value(x) bulk collect into l_result from table(l_annotations) x order by x.position asc; return l_result; + end parse_object_annotations; function parse_object_annotations(a_source_lines dbms_preprocessor.source_lines_t, a_object_type varchar2) return ut_annotations is l_processed_lines dbms_preprocessor.source_lines_t; - l_source clob; + l_source dbms_preprocessor.source_lines_t; l_annotations ut_annotations := ut_annotations(); ex_package_is_wrapped exception; pragma exception_init(ex_package_is_wrapped, -24241); @@ -233,13 +241,11 @@ create or replace package body ut_annotation_parser as else l_processed_lines := sys.dbms_preprocessor.get_post_processed_source(a_source_lines); end if; - --convert to clob for i in 1..l_processed_lines.count loop - ut_utils.append_to_clob(l_source, replace(l_processed_lines(i), chr(13)||chr(10), chr(10))); + l_source(i) := replace(l_processed_lines(i), chr(13)||chr(10), chr(10)); end loop; --parse annotations l_annotations := parse_object_annotations(l_source); - dbms_lob.freetemporary(l_source); exception when ex_package_is_wrapped or source_text_is_empty then null; diff --git a/source/core/annotations/ut_annotation_parser.pks b/source/core/annotations/ut_annotation_parser.pks index 2f474c883..8182acbc7 100644 --- a/source/core/annotations/ut_annotation_parser.pks +++ b/source/core/annotations/ut_annotation_parser.pks @@ -29,16 +29,5 @@ create or replace package ut_annotation_parser authid current_user as */ function parse_object_annotations(a_source_lines dbms_preprocessor.source_lines_t, a_object_type varchar2) return ut_annotations; - - /** - * - * @private - * Parses source code and converts it to annotations - * - * @param a_source_lines ordered lines of source code to be parsed - * @return array containing annotations - */ - function parse_object_annotations(a_source clob) return ut_annotations; - end; / diff --git a/source/core/ut_utils.pkb b/source/core/ut_utils.pkb index 81d47221b..ccde8f966 100644 --- a/source/core/ut_utils.pkb +++ b/source/core/ut_utils.pkb @@ -23,8 +23,10 @@ create or replace package body ut_utils is gc_invalid_xml_char constant varchar2(50) := '[^_[:alnum:]\.-]'; gc_full_valid_xml_name constant varchar2(50) := '^([[:alpha:]])([_[:alnum:]\.-])*$'; gc_owner_hash constant integer(11) := dbms_utility.get_hash_value( ut_owner(), 0, power(2,31)-1); + gc_open_chars constant varchar2(4):= '[{(<'; + gc_close_chars constant varchar2(4):= ']})>'; + gc_max_plsql_source_len constant integer := 32767; - function surround_with(a_value varchar2, a_quote_char varchar2) return varchar2 is begin return case when a_quote_char is not null then a_quote_char||a_value||a_quote_char else a_value end; @@ -658,94 +660,234 @@ create or replace package body ut_utils is return l_result; end; - function replace_multiline_comments(a_source clob) return clob is - l_result clob; - l_ml_comment_start binary_integer := 1; - l_comment_start binary_integer := 1; - l_text_start binary_integer := 1; - l_escaped_text_start binary_integer := 1; - l_escaped_text_end_char varchar2(1 char); - l_end binary_integer := 1; - l_ml_comment clob; - l_newlines_count binary_integer; - l_offset binary_integer := 1; - l_length binary_integer := coalesce(dbms_lob.getlength(a_source), 0); - begin - l_ml_comment_start := instr(a_source,'/*'); - l_comment_start := instr(a_source,'--'); - l_text_start := instr(a_source,''''); - l_escaped_text_start := instr(a_source,q'[q']'); - while l_offset > 0 and l_ml_comment_start > 0 loop - - if l_ml_comment_start > 0 and (l_ml_comment_start < l_comment_start or l_comment_start = 0) - and (l_ml_comment_start < l_text_start or l_text_start = 0)and (l_ml_comment_start < l_escaped_text_start or l_escaped_text_start = 0) - then - l_end := instr(a_source,'*/',l_ml_comment_start+2); - append_to_clob(l_result, dbms_lob.substr(a_source, l_ml_comment_start-l_offset, l_offset)); + function scan_line(a_line in varchar2, a_in_ml_comment in out boolean) return varchar2 is + -- Normal scan + l_remaining varchar2(32767) := a_line; + l_line varchar2(32767); + l_ml_start binary_integer; + l_comment_start binary_integer; + l_text_start binary_integer; + l_eq_text_start binary_integer; + l_eq_end_char varchar2(1 char); + l_pos binary_integer; + l_end binary_integer; + l_ml_end binary_integer; + begin + if a_in_ml_comment then + l_ml_end := instr(l_remaining, '*/'); + if l_ml_end > 0 then + a_in_ml_comment := false; + l_remaining := substr(l_remaining, l_ml_end + 2); + else + return null; + end if; + end if; + + loop + exit when l_remaining is null; + l_ml_start := instr(l_remaining, '/*'); + l_comment_start := instr(l_remaining, '--'); + l_text_start := instr(l_remaining, ''''); + -- q' always puts ' at l_text_start; just check the char immediately before it + l_eq_text_start := case + when l_text_start > 1 and substr(l_remaining, l_text_start - 1, 1) = 'q' + then l_text_start - 1 + else 0 + end; + -- Sentinel gc_max_plsql_source_len means "not present"; 32767 is beyond any VARCHAR2 position + l_pos := least( + case when l_ml_start > 0 then l_ml_start else gc_max_plsql_source_len end, + case when l_comment_start > 0 then l_comment_start else gc_max_plsql_source_len end, + case when l_text_start > 0 then l_text_start else gc_max_plsql_source_len end, + case when l_eq_text_start > 0 then l_eq_text_start else gc_max_plsql_source_len end + ); + + if l_pos = gc_max_plsql_source_len then + l_line := l_line || l_remaining; + exit; + end if; + + l_line := l_line || substr(l_remaining, 1, l_pos - 1); + l_remaining := substr(l_remaining, l_pos); + -- l_remaining now starts exactly at the token; all branch offsets below are relative to 1 + if l_pos = l_eq_text_start then + -- q-quoted string: l_remaining starts at 'q', delimiter is at position 3 + l_eq_end_char := translate(substr(l_remaining, 3, 1), gc_open_chars, gc_close_chars); + l_end := instr(l_remaining, l_eq_end_char || '''', 4); if l_end > 0 then - l_ml_comment := substr(a_source, l_ml_comment_start, l_end-l_ml_comment_start); - l_newlines_count := length( l_ml_comment ) - length( translate( l_ml_comment, 'a'||chr(10), 'a') ); - if l_newlines_count > 0 then - append_to_clob(l_result, lpad( chr(10), l_newlines_count, chr(10) ) ); - end if; - l_end := l_end + 2; + l_line := l_line || substr(l_remaining, 1, l_end + 1); + l_remaining := substr(l_remaining, l_end + 2); + else + l_line := l_line || l_remaining; + exit; end if; - else - if l_comment_start > 0 and (l_comment_start < l_ml_comment_start or l_ml_comment_start = 0) - and (l_comment_start < l_text_start or l_text_start = 0) and (l_comment_start < l_escaped_text_start or l_escaped_text_start = 0) - then - l_end := instr(a_source,chr(10),l_comment_start+2); - if l_end > 0 then - l_end := l_end + 1; - end if; - elsif l_text_start > 0 and (l_text_start < l_ml_comment_start or l_ml_comment_start = 0) - and (l_text_start < l_comment_start or l_comment_start = 0) and (l_text_start < l_escaped_text_start or l_escaped_text_start = 0) - then - l_end := instr(a_source,q'[']',l_text_start+1); - - --skip double quotes while searching for end of quoted text - while l_end > 0 and l_end = instr(a_source,q'['']',l_text_start+1) loop - l_end := instr(a_source,q'[']',l_end+1); - end loop; - if l_end > 0 then - l_end := l_end + 1; + elsif l_pos = l_ml_start then + -- Multi-line comment: l_remaining starts at '/*', so end search starts at 3 + l_ml_end := instr(l_remaining, '*/', 3); + if l_ml_end > 0 then + l_remaining := substr(l_remaining, l_ml_end + 2); + else + a_in_ml_comment := true; + -- preserve trailing newline if present — it belongs to this line, not the comment + if substr(l_remaining, -1) = chr(10) then + l_line := l_line || chr(10); end if; + return l_line; + end if; + + elsif l_pos = l_comment_start then + -- Single-line comment: everything from here is comment, keep and stop + l_line := l_line || l_remaining; + return l_line; - elsif l_escaped_text_start > 0 and (l_escaped_text_start < l_ml_comment_start or l_ml_comment_start = 0) - and (l_escaped_text_start < l_comment_start or l_comment_start = 0) and (l_escaped_text_start < l_text_start or l_text_start = 0) - then - --translate char "[" from the start of quoted text "q'[someting]'" into "]" - l_escaped_text_end_char := translate( substr(a_source, l_escaped_text_start + 2, 1), '[{(<', ']})>'); - l_end := instr(a_source,l_escaped_text_end_char||'''',l_escaped_text_start + 3 ); - if l_end > 0 then - l_end := l_end + 2; + else + -- Regular string literal: l_remaining starts at the opening quote + -- scan from position 2 to skip the opening quote + l_end := 2; + loop + l_end := instr(l_remaining, '''', l_end); + exit when l_end = 0; + if substr(l_remaining, l_end, 2) = '''''' then + l_end := l_end + 2; -- skip escaped quote pair + else + exit; -- real closing quote end if; - end if; + end loop; - if l_end = 0 then - append_to_clob(l_result, substr(a_source, l_offset, l_length-l_offset)); + if l_end > 0 then + l_line := l_line || substr(l_remaining, 1, l_end); + l_remaining := substr(l_remaining, l_end + 1); else - append_to_clob(l_result, substr(a_source, l_offset, l_end-l_offset)); + l_line := l_line || l_remaining; + exit; end if; + end if; - l_offset := l_end; - if l_offset >= l_ml_comment_start then - l_ml_comment_start := instr(a_source,'/*',l_offset); + end loop; + + return l_line; + end; + + function replace_multiline_comments(a_source dbms_preprocessor.source_lines_t) + return dbms_preprocessor.source_lines_t + is + l_result dbms_preprocessor.source_lines_t; + l_line varchar2(32767); + l_remaining varchar2(32767); + l_in_ml_comment boolean := false; + l_ml_end binary_integer; + l_has_ml_comment boolean := false; + begin + if a_source.count = 0 then + return a_source; + end if; + + -- Fast pre-scan to check for presence of multi-line comments; if none, return original source unmodified + for i in 1 .. a_source.count loop + if instr(a_source(i), '/*') > 0 then + l_has_ml_comment := true; + exit; end if; - if l_offset >= l_comment_start then - l_comment_start := instr(a_source,'--',l_offset); + end loop; + + if not l_has_ml_comment then + return a_source; + end if; + + for i in 1 .. a_source.count loop + l_line := a_source(i); + + -- Fast path: inside multi-line comment + if l_in_ml_comment then + l_ml_end := instr(l_line, '*/'); + if l_ml_end > 0 then + l_in_ml_comment := false; + l_line := substr(l_line, l_ml_end + 2); + -- fall through to normal scan + else + l_result(i) := ''; + continue; + end if; end if; - if l_offset >= l_text_start then - l_text_start := instr(a_source,'''',l_offset); + + -- Fast path: no special tokens on this line + if instr(l_line, '/') = 0 + and instr(l_line, '-') = 0 + and instr(l_line, '''') = 0 + then + l_result(i) := l_line; + continue; end if; - if l_offset >= l_escaped_text_start then - l_escaped_text_start := instr(a_source,q'[q']',l_offset); + + -- Normal scan + l_remaining := l_line; + l_line := scan_line(l_remaining, l_in_ml_comment); + if l_line is null then + l_line := ''; end if; + l_result(i) := l_line; end loop; - append_to_clob(l_result, substr(a_source, l_end)); + return l_result; - end; + end replace_multiline_comments; + + function strip_create_header_lines(a_source dbms_preprocessor.source_lines_t) + return dbms_preprocessor.source_lines_t + is + l_result dbms_preprocessor.source_lines_t := a_source; + l_rebased dbms_preprocessor.source_lines_t; + l_create_line pls_integer; + l_header_line pls_integer; + l_header_pos pls_integer := 0; + begin + if l_result.count = 0 then + return l_result; + end if; + + -- remove comment lines that contain "-- create or replace" and find first CREATE + for i in 1..l_result.count loop + l_result(i) := regexp_replace(l_result(i), '^.*[-]{2,}\s*create(\s+or\s+replace).*$', null, 1, 1, 'i'); + if l_create_line is null and regexp_like(l_result(i), '(^|[[:space:]])create([[:space:]]|$)', 'i') then + l_create_line := i; + end if; + end loop; + + -- find first occurrence of object keyword after CREATE (may be on later line) + if l_create_line is not null then + for i in l_create_line..l_result.count loop + l_header_pos := regexp_instr( + l_result(i), + '(^|[[:space:]])(package|type|procedure|function)([[:space:]]|$)', + 1, 1, 0, 'i', 2 + ); + if l_header_pos > 0 then + l_header_line := i; + exit; + end if; + end loop; + + if l_header_line is not null then + -- keep from keyword onward on the header line + l_result(l_header_line) := substr(l_result(l_header_line), l_header_pos); + -- remove "OWNER." from create or replace statement. + -- Owner is not supported along with AUTHID - see issue https://github.com/utPLSQL/utPLSQL/issues/1088 + l_result(l_header_line) := regexp_replace( + l_result(l_header_line), + '^(package|type|procedure|function)\s+("?[[:alpha:]][[:alnum:]$#_]*"?\.)(.*)', + '\1 \3', 1, 1, 'ni' + ); + + -- rebase so header line becomes line 1 (matches preprocessor expectations) + for i in l_header_line .. l_result.count loop + l_rebased(i - l_header_line + 1) := l_result(i); + end loop; + return l_rebased; + end if; + end if; + + return l_result; + end strip_create_header_lines; function get_child_reporters(a_for_reporters ut_reporters_info := null) return ut_reporters_info is l_for_reporters ut_reporters_info := a_for_reporters; diff --git a/source/core/ut_utils.pks b/source/core/ut_utils.pks index a0d1e612c..43b28e6eb 100644 --- a/source/core/ut_utils.pks +++ b/source/core/ut_utils.pks @@ -405,7 +405,15 @@ create or replace package ut_utils authid definer is /** * Replaces multi-line comments in given source-code with empty lines */ - function replace_multiline_comments(a_source clob) return clob; + function replace_multiline_comments(a_source dbms_preprocessor.source_lines_t) + return dbms_preprocessor.source_lines_t; + + /** + * Strips the CREATE header (possibly split across lines) so source starts at + * package/type/procedure/function keyword, preserving line numbers. + */ + function strip_create_header_lines(a_source dbms_preprocessor.source_lines_t) + return dbms_preprocessor.source_lines_t; /** * Returns list of sub-type reporters for given list of super-type reporters diff --git a/test/ut3_tester/core/annotations/test_annotation_parser.pkb b/test/ut3_tester/core/annotations/test_annotation_parser.pkb index bc789c377..beb526863 100644 --- a/test/ut3_tester/core/annotations/test_annotation_parser.pkb +++ b/test/ut3_tester/core/annotations/test_annotation_parser.pkb @@ -1,27 +1,28 @@ create or replace package body test_annotation_parser is procedure test_proc_comments is - l_source clob; + l_source dbms_preprocessor.source_lines_t; l_actual ut3_develop.ut_annotations; l_expected ut3_develop.ut_annotations; - begin - l_source := 'PACKAGE test_tt AS - -- %suite - -- %displayname(Name of suite) - -- %suitepath(all.globaltests) - - -- %ann1(Name of suite) - -- wrong line - -- %ann2(some_value) - procedure foo; - END;'; + --Arrange + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %suite' || chr(10), + ' -- %displayname(Name of suite)' || chr(10), + ' -- %suitepath(all.globaltests)' || chr(10), + '' || chr(10), + ' -- %ann1(Name of suite)' || chr(10), + ' -- wrong line' || chr(10), + ' -- %ann2(some_value)' || chr(10), + ' procedure foo;' || chr(10), + ' END;' || chr(10) + )); --Act - l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source); + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); --Assert - l_expected := ut3_develop.ut_annotations( ut3_develop.ut_annotation(2,'suite',null, null), ut3_develop.ut_annotation(3,'displayname','Name of suite',null), @@ -34,30 +35,33 @@ create or replace package body test_annotation_parser is end; procedure include_floating_annotations is - l_source clob; + l_source dbms_preprocessor.source_lines_t; l_actual ut3_develop.ut_annotations; l_expected ut3_develop.ut_annotations; begin - l_source := 'PACKAGE test_tt AS - -- %suite - -- %displayname(Name of suite) - -- %suitepath(all.globaltests) - - -- %ann1(Name of suite) - -- %ann2(all.globaltests) - - --%test - procedure foo; - - -- %ann3(Name of suite) - -- %ann4(all.globaltests) - - --%test - procedure bar; - END;'; + --Arrange + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %suite' || chr(10), + ' -- %displayname(Name of suite)' || chr(10), + ' -- %suitepath(all.globaltests)' || chr(10), + '' || chr(10), + ' -- %ann1(Name of suite)' || chr(10), + ' -- %ann2(all.globaltests)' || chr(10), + '' || chr(10), + ' --%test' || chr(10), + ' procedure foo;' || chr(10), + '' || chr(10), + ' -- %ann3(Name of suite)' || chr(10), + ' -- %ann4(all.globaltests)' || chr(10), + '' || chr(10), + ' --%test' || chr(10), + ' procedure bar;' || chr(10), + ' END;' || chr(10) + )); --Act - l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source); + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); --Assert l_expected := ut3_develop.ut_annotations( @@ -77,40 +81,41 @@ create or replace package body test_annotation_parser is end; procedure parse_complex_with_functions is - l_source clob; + l_source dbms_preprocessor.source_lines_t; l_actual ut3_develop.ut_annotations; l_expected ut3_develop.ut_annotations; begin - l_source := 'PACKAGE test_tt AS - -- %suite - -- %displayname(Name of suite) - -- %suitepath(all.globaltests) - - --%test - procedure foo; - - - --%beforeeach - procedure foo2; - - --test comment - -- wrong comment - - - /* - describtion of the procedure - */ - --%beforeeach(key=testval) - PROCEDURE foo3(a_value number default null); - - --%all - function foo4(a_val number default null - , a_par varchar2 default := ''asdf''); - END;'; + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %suite' || chr(10), + ' -- %displayname(Name of suite)' || chr(10), + ' -- %suitepath(all.globaltests)' || chr(10), + '' || chr(10), + ' --%test' || chr(10), + ' procedure foo;' || chr(10), + '' || chr(10), + '' || chr(10), + ' --%beforeeach' || chr(10), + ' procedure foo2;' || chr(10), + '' || chr(10), + ' --test comment' || chr(10), + ' -- wrong comment' || chr(10), + '' || chr(10), + '' || chr(10), + '/*' || chr(10), + ' describtion of the procedure' || chr(10), + ' */' || chr(10), + ' --%beforeeach(key=testval)' || chr(10), + ' PROCEDURE foo3(a_value number default null);' || chr(10), + '' || chr(10), + ' --%all' || chr(10), + ' function foo4(a_val number default null' || chr(10), + ' , a_par varchar2 default := ''asdf'');' || chr(10), + 'END;')); --Act - l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source); + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); --Assert l_expected := ut3_develop.ut_annotations( @@ -128,21 +133,24 @@ create or replace package body test_annotation_parser is end; procedure no_procedure_annotation is - l_source clob; + l_source dbms_preprocessor.source_lines_t; l_actual ut3_develop.ut_annotations; l_expected ut3_develop.ut_annotations; begin - l_source := 'PACKAGE test_tt AS - -- %suite - -- %displayname(Name of suite) - -- %suitepath(all.globaltests) - - procedure foo; - END;'; + --Arrange + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %suite' || chr(10), + ' -- %displayname(Name of suite)' || chr(10), + ' -- %suitepath(all.globaltests)' || chr(10), + '' || chr(10), + ' procedure foo;' || chr(10), + ' END;' || chr(10) + )); --Act - l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source); + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); --Assert l_expected := ut3_develop.ut_annotations( @@ -156,21 +164,24 @@ create or replace package body test_annotation_parser is end; procedure parse_accessible_by is - l_source clob; + l_source dbms_preprocessor.source_lines_t; l_actual ut3_develop.ut_annotations; l_expected ut3_develop.ut_annotations; begin - l_source := 'PACKAGE test_tt accessible by (foo) AS - -- %suite - -- %displayname(Name of suite) - -- %suitepath(all.globaltests) - - procedure foo; - END;'; + --Arrange + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt accessible by (foo) AS' || chr(10), + ' -- %suite' || chr(10), + ' -- %displayname(Name of suite)' || chr(10), + ' -- %suitepath(all.globaltests)' || chr(10), + '' || chr(10), + ' procedure foo;' || chr(10), + ' END;' || chr(10) + )); --Act - l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source); + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); --Assert l_expected := ut3_develop.ut_annotations( @@ -184,24 +195,27 @@ create or replace package body test_annotation_parser is end; procedure complex_package_declaration is - l_source clob; + l_source dbms_preprocessor.source_lines_t; l_actual ut3_develop.ut_annotations; l_expected ut3_develop.ut_annotations; begin - l_source := 'PACKAGE test_tt - ACCESSIBLE BY (calling_proc) - authid current_user - AS - -- %suite - -- %displayname(Name of suite) - -- %suitepath(all.globaltests) - - procedure foo; - END;'; + --Arrange + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt' || chr(10), + ' ACCESSIBLE BY (calling_proc)' || chr(10), + ' authid current_user' || chr(10), + ' AS' || chr(10), + ' -- %suite' || chr(10), + ' -- %displayname(Name of suite)' || chr(10), + ' -- %suitepath(all.globaltests)' || chr(10), + '' || chr(10), + ' procedure foo;' || chr(10), + ' END;' || chr(10) + )); --Act - l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source); + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); --Assert l_expected := ut3_develop.ut_annotations( @@ -215,21 +229,24 @@ create or replace package body test_annotation_parser is end; procedure complex_text is - l_source clob; + l_source dbms_preprocessor.source_lines_t; l_actual ut3_develop.ut_annotations; l_expected ut3_develop.ut_annotations; begin - l_source := 'PACKAGE test_tt AS - -- %suite - --%displayname(name = Name of suite) - -- %suitepath(key=all.globaltests,key2=foo,"--%some text") - - procedure foo; - END;'; + --Arrange + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %suite' || chr(10), + ' --%displayname(name = Name of suite)' || chr(10), + ' -- %suitepath(key=all.globaltests,key2=foo,"--%some text")' || chr(10), + '' || chr(10), + ' procedure foo;' || chr(10), + ' END;' || chr(10) + )); --Act - l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source); + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); --Assert l_expected := ut3_develop.ut_annotations( @@ -243,26 +260,29 @@ create or replace package body test_annotation_parser is end; procedure ignore_annotations_in_comments is - l_source clob; + l_source dbms_preprocessor.source_lines_t; l_actual ut3_develop.ut_annotations; l_expected ut3_develop.ut_annotations; begin - l_source := 'PACKAGE test_tt AS - /* - Some comment - -- inlined - -- %ignored - */ - -- %suite - --%displayname(Name of suite) - -- %suitepath(all.globaltests) - - procedure foo; - END;'; + --Arrange + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' /*' || chr(10), + ' Some comment' || chr(10), + ' -- inlined' || chr(10), + ' -- %ignored' || chr(10), + ' */' || chr(10), + ' -- %suite' || chr(10), + ' --%displayname(Name of suite)' || chr(10), + ' -- %suitepath(all.globaltests)' || chr(10), + '' || chr(10), + ' procedure foo;' || chr(10), + ' END;' || chr(10) + )); --Act - l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source); + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); --Assert l_expected := ut3_develop.ut_annotations( @@ -311,17 +331,19 @@ v58yvbLAXLi9gYHwoIvAgccti+Cmpg0DKLY= end; procedure brackets_in_desc is - - l_source clob; + l_source dbms_preprocessor.source_lines_t; l_actual ut3_develop.ut_annotations; l_expected ut3_develop.ut_annotations; begin - l_source := 'PACKAGE test_tt AS - -- %suite(Name of suite (including some brackets) and some more text) -END;'; + --Arrange + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %suite(Name of suite (including some brackets) and some more text)' || chr(10), + 'END;' || chr(10) + )); --Act - l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source); + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); --Assert l_expected := ut3_develop.ut_annotations( @@ -332,24 +354,27 @@ END;'; end; procedure test_space_before_annot_params is - l_source clob; + l_source dbms_preprocessor.source_lines_t; l_actual ut3_develop.ut_annotations; - l_expected ut3_develop.ut_annotations; + l_expected ut3_develop.ut_annotations; begin - l_source := 'PACKAGE test_tt AS - /* - Some comment - -- inlined - */ - -- %suite - -- %suitepath (all.globaltests) - - procedure foo; -END;'; + --Arrange + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' /*' || chr(10), + ' Some comment' || chr(10), + ' -- inlined' || chr(10), + ' */' || chr(10), + ' -- %suite' || chr(10), + ' -- %suitepath (all.globaltests)'|| chr(10), + '' || chr(10), + ' procedure foo;' || chr(10), + 'END;' || chr(10) + )); --Act - l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source); + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); --Assert l_expected := ut3_develop.ut_annotations( @@ -362,18 +387,21 @@ END;'; procedure test_windows_newline as - l_source clob; + l_source dbms_preprocessor.source_lines_t; l_actual ut3_develop.ut_annotations; l_expected ut3_develop.ut_annotations; begin - l_source := 'PACKAGE test_tt AS - -- %suite - -- %displayname(Name of suite)' || chr(13) || chr(10) - || ' -- %suitepath(all.globaltests) - END;'; + --Arrange + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %suite' || chr(10), + ' -- %displayname(Name of suite)' || chr(13) || chr(10), + ' -- %suitepath(all.globaltests)' || chr(10), + ' END;' || chr(10) + )); --Act - l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source); + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); --Assert l_expected := ut3_develop.ut_annotations( @@ -387,21 +415,24 @@ END;'; procedure test_annot_very_long_name as - l_source clob; - l_actual ut3_develop.ut_annotations; - l_expected ut3_develop.ut_annotations; + l_source dbms_preprocessor.source_lines_t; + l_actual ut3_develop.ut_annotations; + l_expected ut3_develop.ut_annotations; begin - l_source := 'PACKAGE very_long_procedure_name_valid_for_oracle_12_so_utPLSQL_should_allow_it_definitely_well_still_not_reached_128_but_wait_we_did_it AS - -- %suite - -- %displayname(Name of suite) - -- %suitepath(all.globaltests) - - --%test - procedure very_long_procedure_name_valid_for_oracle_12_so_utPLSQL_should_allow_it_definitely_well_still_not_reached_128_but_wait_we_dit_it; - END;'; + --Arrange + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE very_long_procedure_name_valid_for_oracle_12_so_utPLSQL_should_allow_it_definitely_well_still_not_reached_128_but_wait_we_did_it AS' || chr(10), + ' -- %suite' || chr(10), + ' -- %displayname(Name of suite)' || chr(10), + ' -- %suitepath(all.globaltests)' || chr(10), + '' || chr(10), + ' --%test' || chr(10), + ' procedure very_long_procedure_name_valid_for_oracle_12_so_utPLSQL_should_allow_it_definitely_well_still_not_reached_128_but_wait_we_dit_it;' || chr(10), + ' END;' || chr(10) + )); --Act - l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source); + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); --Assert l_expected := ut3_develop.ut_annotations( @@ -415,30 +446,33 @@ END;'; end; procedure test_upper_annot is - l_source clob; + l_source dbms_preprocessor.source_lines_t; l_actual ut3_develop.ut_annotations; l_expected ut3_develop.ut_annotations; begin - l_source := 'PACKAGE test_tt AS - -- %SUITE - -- %DISPLAYNAME(Name of suite) - -- %SUITEPATH(all.globaltests) - - -- %ANN1(Name of suite) - -- %ANN2(all.globaltests) - - --%TEST - procedure foo; - - -- %ANN3(Name of suite) - -- %ANN4(all.globaltests) - - --%TEST - procedure bar; - END;'; + --Arrange + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %SUITE' || chr(10), + ' -- %DISPLAYNAME(Name of suite)' || chr(10), + ' -- %SUITEPATH(all.globaltests)' || chr(10), + '' || chr(10), + ' -- %ANN1(Name of suite)' || chr(10), + ' -- %ANN2(all.globaltests)' || chr(10), + '' || chr(10), + ' --%TEST' || chr(10), + ' procedure foo;' || chr(10), + '' || chr(10), + ' -- %ANN3(Name of suite)' || chr(10), + ' -- %ANN4(all.globaltests)' || chr(10), + '' || chr(10), + ' --%TEST' || chr(10), + ' procedure bar;' || chr(10), + ' END;' || chr(10) + )); --Act - l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source); + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); --Assert l_expected := ut3_develop.ut_annotations( @@ -457,5 +491,414 @@ END;'; end; + ------------------------------------------------------------ + -- replace_multiline_comments coverage tests + ------------------------------------------------------------ + + procedure test_rmc_empty_source is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + begin + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(l_result.count).to_equal(0); + end; + + procedure test_rmc_no_ml_comment_marker is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + begin + --Arrange + l_input := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'procedure foo is' || chr(10), + 'begin' || chr(10), + ' null;' || chr(10), + 'end;' || chr(10) + )); + + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(ut3_tester_helper.main_helper.lines_to_str(l_result)).to_equal(ut3_tester_helper.main_helper.lines_to_str(l_input)); + end; + + procedure test_rmc_line_inside_ml_comment is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + begin + --Arrange + l_input := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'x := 1; /* start' || chr(10), + 'this whole line is comment' || chr(10), + 'end comment */ x := 2;' || chr(10) + )); + + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(l_result(1)).to_equal('x := 1; ' || chr(10)); + ut.expect(l_result(2)).to_equal(''); + ut.expect(l_result(3)).to_equal(' x := 2;' || chr(10)); + end; + + procedure test_rmc_ml_comment_closed_midline is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + begin + --Arrange + l_input := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + '/* open' || chr(10), + 'still inside' || chr(10), + '*/ code /* remove this too */ kept' || chr(10) + )); + + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(l_result(2)).to_equal(''); + ut.expect(l_result(3)).to_equal(' code kept' || chr(10)); + end; + + procedure test_rmc_fast_path_no_tokens is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + begin + --Arrange + -- line 2 has none of / - ' so hits fast path B; line 1 needed to pass pre-scan + l_input := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + '/* comment */' || chr(10), + 'begin' || chr(10) + )); + + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(l_result(2)).to_equal('begin' || chr(10)); + end; + + procedure test_rmc_ml_comment_single_line is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + begin + --Arrange + l_input := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'x := /* inline comment */ 42;' || chr(10) + )); + + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(l_result(1)).to_equal('x := 42;' || chr(10)); + end; + + procedure test_rmc_single_line_comment_preserved is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + begin + --Arrange + l_input := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + '/* marker */' || chr(10), + ' -- %test annotation' || chr(10) + )); + + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(l_result(2)).to_equal(' -- %test annotation' || chr(10)); + end; + + procedure test_rmc_string_literal_protects_markers is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + begin + --Arrange + l_input := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'v := ''val /* not a comment */ here'';' || chr(10) + )); + + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(l_result(1)).to_equal('v := ''val /* not a comment */ here'';' || chr(10)); + end; + + procedure test_rmc_string_literal_escaped_quotes is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + begin + --Arrange + l_input := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'v := ''it''''s a /* test */'';' || chr(10) + )); + + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(l_result(1)).to_equal('v := ''it''''s a /* test */'';' || chr(10)); + end; + + procedure test_rmc_q_quoted_string_literal is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + begin + --Arrange + l_input := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'v := q''[/* not a comment */]'';' || chr(10) + )); + + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(l_result(1)).to_equal('v := q''[/* not a comment */]'';' || chr(10)); + end; + + procedure test_rmc_unclosed_string_literal is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + begin + --Arrange + l_input := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'v := ''hello /* inside unclosed' || chr(10) + )); + + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(l_result(1)).to_equal('v := ''hello /* inside unclosed' || chr(10)); + end; + + procedure test_rmc_unclosed_q_string is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + c_line constant varchar2(100) := q'(v := q'[/* unclosed q-string)' || chr(10); + begin + --Arrange + l_input := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list(c_line)); + + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(l_result(1)).to_equal(c_line); + end; + + procedure test_rmc_multiple_ml_comments_one_line is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + begin + --Arrange + l_input := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'a /* one */ := /* two */ 1;' || chr(10) + )); + + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(l_result(1)).to_equal('a := 1;' || chr(10)); + end; + + procedure test_rmc_comment_after_string_with_slash is + l_input dbms_preprocessor.source_lines_t; + l_result dbms_preprocessor.source_lines_t; + begin + --Arrange + l_input := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'v := ''a/b''; -- this is /* not */ a ml comment' || chr(10) + )); + + --Act + l_result := ut3_develop.ut_utils.replace_multiline_comments(l_input); + + --Assert + ut.expect(l_result(1)).to_equal('v := ''a/b''; -- this is /* not */ a ml comment' || chr(10)); + end; + + procedure test_multiline_proc_header_lines is + l_source dbms_preprocessor.source_lines_t; + l_actual ut3_develop.ut_annotations; + l_expected ut3_develop.ut_annotations; + begin + --Arrange + -- procedure header spans multiple lines before terminating ; + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %suite' || chr(10), + '' || chr(10), + ' --%test' || chr(10), + ' procedure foo(' || chr(10), + ' a_param1 varchar2' || chr(10), + ' ,a_param2 number default null' || chr(10), + ' );' || chr(10), + ' END;' || chr(10) + )); + + --Act + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); + + --Assert + l_expected := ut3_develop.ut_annotations( + ut3_develop.ut_annotation( 2, 'suite', null, null ), + ut3_develop.ut_annotation( 4, 'test', null, 'foo' ) + ); + + ut.expect(anydata.convertCollection(l_actual)).to_equal(anydata.convertCollection(l_expected)); + end; + + procedure test_non_comment_line_resets_block_lines is + l_source dbms_preprocessor.source_lines_t; + l_actual ut3_develop.ut_annotations; + l_expected ut3_develop.ut_annotations; + begin + --Arrange + -- a non-comment non-proc line between annotation and procedure breaks the association + -- %test becomes a floating top-level annotation with no subobject + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %suite' || chr(10), + '' || chr(10), + ' --%test' || chr(10), + ' pragma serially_reusable;' || chr(10), + ' procedure foo;' || chr(10), + ' END;' || chr(10) + )); + + --Act + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); + + --Assert + -- %test is NOT associated with foo — accumulator was reset by pragma line + -- it surfaces as a top-level annotation with no subobject_name + l_expected := ut3_develop.ut_annotations( + ut3_develop.ut_annotation( 2, 'suite', null, null ), + ut3_develop.ut_annotation( 4, 'test', null, null ) + ); + + ut.expect(anydata.convertCollection(l_actual)).to_equal(anydata.convertCollection(l_expected)); + end; + + procedure test_annotation_not_at_line_start_ignored_lines is + l_source dbms_preprocessor.source_lines_t; + l_actual ut3_develop.ut_annotations; + l_expected ut3_develop.ut_annotations; + begin + --Arrange + -- code before -- means it is not at start of line and must be ignored + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %suite' || chr(10), + ' x := 1; -- %not_an_annotation' || chr(10), + ' procedure foo;' || chr(10), + ' END;' || chr(10) + )); + + --Act + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); + + --Assert + l_expected := ut3_develop.ut_annotations( + ut3_develop.ut_annotation( 2, 'suite', null, null ) + ); + + ut.expect(anydata.convertCollection(l_actual)).to_equal(anydata.convertCollection(l_expected)); + end; + + procedure test_whitespace_line_between_comment_and_proc_lines is + l_source dbms_preprocessor.source_lines_t; + l_actual ut3_develop.ut_annotations; + l_expected ut3_develop.ut_annotations; + begin + --Arrange + -- space-only line between annotation comment and procedure declaration is tolerated + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %suite' || chr(10), + '' || chr(10), + ' --%test' || chr(10), + ' ' || chr(10), + ' procedure foo;' || chr(10), + ' END;' || chr(10) + )); + + --Act + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); + + --Assert + l_expected := ut3_develop.ut_annotations( + ut3_develop.ut_annotation( 2, 'suite', null, null ), + ut3_develop.ut_annotation( 4, 'test', null, 'foo' ) + ); + + ut.expect(anydata.convertCollection(l_actual)).to_equal(anydata.convertCollection(l_expected)); + end; + + procedure test_space_between_dashes_and_qualifier_lines is + l_source dbms_preprocessor.source_lines_t; + l_actual ut3_develop.ut_annotations; + l_expected ut3_develop.ut_annotations; + begin + --Arrange + -- spaces between -- and % are skipped and annotation is still recognised + -- annotations are not adjacent to a procedure so they are top-level + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %suite' || chr(10), + ' -- %displayname(my suite)' || chr(10), + '' || chr(10), + ' procedure foo;' || chr(10), + ' END;' || chr(10) + )); + + --Act + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); + + --Assert + l_expected := ut3_develop.ut_annotations( + ut3_develop.ut_annotation( 2, 'suite', null, null ), + ut3_develop.ut_annotation( 3, 'displayname', 'my suite', null ) + ); + + ut.expect(anydata.convertCollection(l_actual)).to_equal(anydata.convertCollection(l_expected)); + end; + + procedure test_percent_not_after_dashes_ignored_lines is + l_source dbms_preprocessor.source_lines_t; + l_actual ut3_develop.ut_annotations; + l_expected ut3_develop.ut_annotations; + begin + --Arrange + -- -- present and % present on line but % is not immediately after -- + -- e.g. a regular comment that happens to mention 100% + l_source := ut3_tester_helper.main_helper.make_source(ut3_develop.ut_varchar2_list( + 'PACKAGE test_tt AS' || chr(10), + ' -- %suite' || chr(10), + ' -- coverage is 100% on this module' || chr(10), + ' procedure foo;' || chr(10), + ' END;' || chr(10) + )); + + --Act + l_actual := ut3_develop.ut_annotation_parser.parse_object_annotations(l_source, 'PACKAGE'); + + --Assert + l_expected := ut3_develop.ut_annotations( + ut3_develop.ut_annotation( 2, 'suite', null, null ) + ); + + ut.expect(anydata.convertCollection(l_actual)).to_equal(anydata.convertCollection(l_expected)); + end; + end test_annotation_parser; -/ +/ \ No newline at end of file diff --git a/test/ut3_tester/core/annotations/test_annotation_parser.pks b/test/ut3_tester/core/annotations/test_annotation_parser.pks index a4fe3ed70..c61c14d3a 100644 --- a/test/ut3_tester/core/annotations/test_annotation_parser.pks +++ b/test/ut3_tester/core/annotations/test_annotation_parser.pks @@ -19,24 +19,57 @@ create or replace package test_annotation_parser is procedure complex_text; --%test(Ignores content of multi-line comments) procedure ignore_annotations_in_comments; - - --%test(Ignores wrapped package and does not raise exception) + -- %test(ignore wrapped package source) procedure ignore_wrapped_package; - --%test(Parses package level annotations with annotation params containing brackets) procedure brackets_in_desc; - --%test(Parses annotation text even with spaces before brackets) procedure test_space_before_annot_params; - -- %test(Parses source-code with Windows-style newline) procedure test_windows_newline; - - -- %test(Parses annotations with very long object names) + -- %test(parse annotations with very long procedure name) procedure test_annot_very_long_name; - -- %test(Parses upper case annotations) procedure test_upper_annot; + -- %test(empty source returns immediately without processing) + procedure test_rmc_empty_source; + -- %test(source with no multiline comment markers returned unchanged) + procedure test_rmc_no_ml_comment_marker; + -- %test(line entirely inside multiline comment is blanked) + procedure test_rmc_line_inside_ml_comment; + -- %test(closing delimiter mid-line falls through to scan remainder) + procedure test_rmc_ml_comment_closed_midline; + -- %test(fast path line with no slash dash or quote copied unchanged) + procedure test_rmc_fast_path_no_tokens; + -- %test(single-line block comment removed on same line) + procedure test_rmc_ml_comment_single_line; + -- %test(single-line comment preserved intact) + procedure test_rmc_single_line_comment_preserved; + -- %test(string literal containing comment markers not stripped) + procedure test_rmc_string_literal_protects_markers; + -- %test(string literal with escaped quotes handled correctly) + procedure test_rmc_string_literal_escaped_quotes; + -- %test(q-quoted string literal protects interior comment markers) + procedure test_rmc_q_quoted_string_literal; + -- %test(unclosed string literal copies remainder and exits scan) + procedure test_rmc_unclosed_string_literal; + -- %test(unclosed q-quoted string copies remainder and exits scan) + procedure test_rmc_unclosed_q_string; + -- %test(multiple block comments on single line all removed) + procedure test_rmc_multiple_ml_comments_one_line; + -- %test(comment markers after string with slash handled correctly) + procedure test_rmc_comment_after_string_with_slash; + --%test(Procedure header spanning multiple lines is handled correctly) + procedure test_multiline_proc_header_lines; + --%test(Non-comment line between annotation and procedure resets association) + procedure test_non_comment_line_resets_block_lines; + --%test(Comment with non-whitespace before dashes is not treated as annotation) + procedure test_annotation_not_at_line_start_ignored_lines; + --%test(Spaces between dashes and annotation qualifier are skipped correctly) + procedure test_space_between_dashes_and_qualifier_lines; + --%test(Percent sign not immediately after dashes is not treated as annotation) + procedure test_percent_not_after_dashes_ignored_lines; + end test_annotation_parser; -/ +/ \ No newline at end of file diff --git a/test/ut3_tester/core/test_ut_utils.pkb b/test/ut3_tester/core/test_ut_utils.pkb index 433987c01..4dc10492b 100644 --- a/test/ut3_tester/core/test_ut_utils.pkb +++ b/test/ut3_tester/core/test_ut_utils.pkb @@ -375,8 +375,8 @@ end;'; procedure replace_multiline_comments is l_source clob; - l_actual clob; - l_expected clob; + l_actual dbms_preprocessor.source_lines_t; + l_expected dbms_preprocessor.source_lines_t; begin --Arrange l_source := q'[ @@ -390,11 +390,11 @@ create or replace package dummy as gv_text2 varchar2(200) := '/* multi-line comment in a multi-line string*/'; - -- ignored start of multi-line comment with multi-byte text � /* - -- ignored end of multi-line comment with multi-byte text � */ + -- ignored start of multi-line comment with multi-byte text ☺ /* + -- ignored end of multi-line comment with multi-byte text ☺ */ /* multi-line comment with - multi-byte characters ��� + multi-byte characters ☺☺☺ in it */ gv_text3 varchar2(200) := 'some text'; /* multiline comment*/ --followed by single-line comment /* multi-line comment in one line*/ @@ -403,7 +403,14 @@ create or replace package dummy as string*/}'; end; ]'; - l_expected := q'[ + + --Act + l_actual := ut3_develop.ut_utils.replace_multiline_comments( + ut3_tester_helper.main_helper.make_source( ut3_develop.ut_utils.clob_to_table(l_source) ) + ); + + --Assert + l_expected := ut3_tester_helper.main_helper.make_source( ut3_develop.ut_utils.clob_to_table( q'[ create or replace package dummy as -- single line comment with disabled /* multi-line comment */ @@ -414,8 +421,8 @@ create or replace package dummy as gv_text2 varchar2(200) := '/* multi-line comment in a multi-line string*/'; - -- ignored start of multi-line comment with multi-byte text � /* - -- ignored end of multi-line comment with multi-byte text � */ + -- ignored start of multi-line comment with multi-byte text ☺ /* + -- ignored end of multi-line comment with multi-byte text ☺ */ ]'||q'[ @@ -426,11 +433,9 @@ create or replace package dummy as in escaped q'multi-line string*/}'; end; -]'; +]' ) ); --Act - l_actual := ut3_develop.ut_utils.replace_multiline_comments(l_source); - --Assert - ut.expect(l_actual).to_equal(l_expected); + ut.expect(ut3_tester_helper.main_helper.lines_to_str(l_actual)).to_equal(ut3_tester_helper.main_helper.lines_to_str(l_expected)); end; procedure int_conv_ds_sec is diff --git a/test/ut3_tester_helper/main_helper.pkb b/test/ut3_tester_helper/main_helper.pkb index f1dadec66..f60ce796c 100644 --- a/test/ut3_tester_helper/main_helper.pkb +++ b/test/ut3_tester_helper/main_helper.pkb @@ -155,6 +155,24 @@ create or replace package body main_helper is begin ut3_develop.ut_session_context.clear_all_context; end; + + function lines_to_str(a_lines dbms_preprocessor.source_lines_t) return varchar2 is + l_result varchar2(32767); + begin + for i in 1 .. a_lines.count loop + l_result := l_result || a_lines(i); + end loop; + return l_result; + end; + + function make_source(a_lines ut3_develop.ut_varchar2_list) return dbms_preprocessor.source_lines_t is + l_result dbms_preprocessor.source_lines_t; + begin + for i in 1 .. a_lines.count loop + l_result(i) := a_lines(i); + end loop; + return l_result; + end; end; / diff --git a/test/ut3_tester_helper/main_helper.pks b/test/ut3_tester_helper/main_helper.pks index 7decad88d..71575e145 100644 --- a/test/ut3_tester_helper/main_helper.pks +++ b/test/ut3_tester_helper/main_helper.pks @@ -47,5 +47,9 @@ create or replace package main_helper is procedure clear_ut_run_context; + function lines_to_str(a_lines dbms_preprocessor.source_lines_t) return varchar2; + + function make_source(a_lines ut3_develop.ut_varchar2_list) return dbms_preprocessor.source_lines_t; + end; /