Changing a code to validate by lines instead of clobs for performance.

utPLSQL · jgebal · Apr 22, 2026 · Mar 9, 2026 · Mar 9, 2026 · Mar 9, 2026
commit 5a217746ef9109cd96185310f92ee8ca6b01e32a
diff --git a/source/core/annotations/ut_annotation_parser.pkb b/source/core/annotations/ut_annotation_parser.pkb
@@ -129,6 +129,78 @@ create or replace package body ut_annotation_parser as
     end loop;
   end add_procedure_annotations;
 
+  procedure add_procedure_annotations(
+    a_annotations in out nocopy ut_annotations,
+    a_source      in            dbms_preprocessor.source_lines_t,
+    a_comments    in out nocopy tt_comment_list
+  ) is
+    l_proc_comments  varchar2(32767);
+    l_proc_name      varchar2(250);
+    l_line           varchar2(32767);
+    l_in_comment_block boolean := false;
+    l_i              binary_integer;
+  begin
+    l_i := 1;
+    while l_i <= a_source.count loop
+      l_line := a_source(l_i);
+
+      -- -----------------------------------------------------------------
+      -- Comment placeholder line: start/continue accumulating a block
+      -- -----------------------------------------------------------------
+      if instr(l_line, '{COMMENT#') > 0 then
+        if l_in_comment_block then
+          l_proc_comments := l_proc_comments || l_line || chr(10);
+        else
+          l_in_comment_block := true;
+          l_proc_comments    := l_line || chr(10);
+        end if;
+        l_i := l_i + 1;
+
+      -- -----------------------------------------------------------------
+      -- Whitespace-only line: allowed between comment block and proc decl
+      -- -----------------------------------------------------------------
+      elsif l_in_comment_block and trim(replace(l_line, chr(9))) is null then
+        l_i := l_i + 1;
+
+      -- -----------------------------------------------------------------
+      -- procedure/function declaration following a comment block
+      -- -----------------------------------------------------------------
+      elsif l_in_comment_block
+            and regexp_like(l_line, '^\s*(procedure|function)\s+', 'i')
+      then
+        -- extract just the identifier name (subexpression 2)
+        l_proc_name := trim(regexp_substr(l_line
+                                        ,pattern       => '^\s*(procedure|function)\s+('||gc_regexp_identifier||')'
+                                        ,modifier      => 'i'
+                                        ,subexpression => 2));
+
+        -- pass accumulated comment placeholders + proc name to add_annotations
+        add_annotations(a_annotations, l_proc_comments, a_comments, l_proc_name);
+
+        -- reset comment block state
+        l_in_comment_block := false;
+        l_proc_comments    := null;
+
+        -- advance past proc header to the terminating ';'
+        -- the header may span multiple lines e.g. with multi-line parameter lists
+        while l_i <= a_source.count loop
+          exit when instr(a_source(l_i), ';') > 0;
+          l_i := l_i + 1;
+        end loop;
+        l_i := l_i + 1;  -- step past the ';' line itself
+
+      -- -----------------------------------------------------------------
+      -- Any other line: reset comment block accumulator
+      -- -----------------------------------------------------------------
+      else
+        l_in_comment_block := false;
+        l_proc_comments    := null;
+        l_i := l_i + 1;
+      end if;
+
+    end loop;
+  end add_procedure_annotations;
+
   function extract_and_replace_comments(a_source in out nocopy clob) return tt_comment_list is
     l_comments         tt_comment_list;
     l_comment_pos      binary_integer;
@@ -175,13 +247,87 @@ create or replace package body ut_annotation_parser as
 
     end loop;
 
-    ut_utils.debug_log(a_source);
+    return l_comments;
+  end extract_and_replace_comments;
+
+  function extract_and_replace_comments(
+    a_source in out nocopy dbms_preprocessor.source_lines_t
+  ) return tt_comment_list is
+    l_comments        tt_comment_list;
+    l_line            varchar2(32767);
+    l_comment_pos     binary_integer;
+    l_comment_replacer varchar2(50);
+  begin
+    for i in 1 .. a_source.count loop
+      l_line := a_source(i);
+
+      -- fast path: skip lines that can't possibly match
+      -- must contain '--' and '%' to be an annotation comment
+      if instr(l_line, '--') = 0 or instr(l_line, gc_annotation_qualifier) = 0 then
+        continue;
+      end if;
+
+      -- find '--' on the line
+      l_comment_pos := instr(l_line, '--');
+
+      -- verify everything before '--' is only spaces/tabs (matches ^ *( |\t)*--)
+      if trim(replace(substr(l_line, 1, l_comment_pos - 1), chr(9))) is not null then
+        continue;
+      end if;
+
+      -- skip '--' and any spaces after it, then check for annotation qualifier '%'
+      l_comment_pos := l_comment_pos + 2;
+      -- skip optional spaces between -- and %
+      while l_comment_pos <= length(l_line)
+        and substr(l_line, l_comment_pos, 1) = ' '
+      loop
+        l_comment_pos := l_comment_pos + 1;
+      end loop;
+
+      -- must start with annotation qualifier at this position
+      if substr(l_line, l_comment_pos, 1) != gc_annotation_qualifier then
+        continue;
+      end if;
+
+      -- extract annotation text (from % to end of line, trimmed)
+      l_comments(i) := trim(substr(l_line, l_comment_pos));
+
+      -- replace line with placeholder, preserving line number in token
+      l_comment_replacer := replace(gc_comment_replacer_patter, '%N%', i);
+      a_source(i) := l_comment_replacer;
+
+    end loop;
+
     return l_comments;
   end extract_and_replace_comments;
 
   ------------------------------------------------------------
   --public definitions
   ------------------------------------------------------------
+  function parse_object_annotations(a_source dbms_preprocessor.source_lines_t) return ut_annotations is
+    l_source           dbms_preprocessor.source_lines_t := a_source;
+    l_comments         tt_comment_list;
+    l_annotations      ut_annotations := ut_annotations();
+    l_result           ut_annotations;
+    l_comment_index    positive;
+  begin
+    l_source := ut_utils.replace_multiline_comments(l_source);
+    l_comments := extract_and_replace_comments(l_source);
+    add_procedure_annotations(l_annotations, l_source, l_comments);
+    delete_processed_comments(l_comments, l_annotations);
+
+    --at this point, only the comments not related to procedures are left, so we process them all as top-level
+    l_comment_index := l_comments.first;
+    while l_comment_index is not null loop
+      add_annotation( l_annotations, l_comment_index, l_comments( l_comment_index ) );
+      l_comment_index := l_comments.next(l_comment_index);
+    end loop;
+
+    select /*+ no_parallel */ value(x) bulk collect into l_result from table(l_annotations) x order by x.position;
+
+    return l_result;
+
+  end parse_object_annotations;
 
   function parse_object_annotations(a_source clob) return ut_annotations is
     l_source           clob := a_source;
@@ -217,7 +363,7 @@ create or replace package body ut_annotation_parser as
 
   function parse_object_annotations(a_source_lines dbms_preprocessor.source_lines_t, a_object_type varchar2) return ut_annotations is
     l_processed_lines dbms_preprocessor.source_lines_t;
-    l_source          clob;
+    l_source          dbms_preprocessor.source_lines_t;
     l_annotations     ut_annotations := ut_annotations();
     ex_package_is_wrapped exception;
     pragma exception_init(ex_package_is_wrapped, -24241);
@@ -235,11 +381,10 @@ create or replace package body ut_annotation_parser as
         end if;
         --convert to clob
         for i in 1..l_processed_lines.count loop
-          ut_utils.append_to_clob(l_source, replace(l_processed_lines(i), chr(13)||chr(10), chr(10)));
+          l_source(i) := replace(l_processed_lines(i), chr(13)||chr(10), chr(10));
         end loop;
         --parse annotations
         l_annotations := parse_object_annotations(l_source);
-        dbms_lob.freetemporary(l_source);
       exception
         when ex_package_is_wrapped or source_text_is_empty then
           null;

diff --git a/source/core/annotations/ut_annotation_parser.pks b/source/core/annotations/ut_annotation_parser.pks
@@ -40,5 +40,7 @@ create or replace package ut_annotation_parser authid current_user as
    */
   function parse_object_annotations(a_source clob) return ut_annotations;
 
+  function parse_object_annotations(a_source dbms_preprocessor.source_lines_t) return ut_annotations;
+
 end;
 /
diff --git a/source/core/ut_utils.pkb b/source/core/ut_utils.pkb
@@ -658,6 +658,182 @@ create or replace package body ut_utils is
     return l_result;
   end;
 
+  function replace_multiline_comments(a_source dbms_preprocessor.source_lines_t)
+    return dbms_preprocessor.source_lines_t
+  is
+    l_result          dbms_preprocessor.source_lines_t;
+    l_line            varchar2(32767);
+    l_remaining       varchar2(32767);
+    l_in_ml_comment   boolean := false;
+    l_ml_end          binary_integer;
+    l_ml_start        binary_integer;
+    l_comment_start   binary_integer;
+    l_text_start      binary_integer;
+    l_eq_text_start   binary_integer;
+    l_eq_end_char     varchar2(1 char);
+    l_pos             binary_integer;
+    l_end             binary_integer;
+    l_token_count     binary_integer;
+    l_has_ml_comment  boolean := false;
+  begin
+
+    -- Guard: empty source
+    if a_source.count = 0 then
+      return a_source;
+    end if;
+
+    -- Fast pre-scan: check if any /* exists at all if not, nothing to do — return source as-is
+    for i in 1 .. a_source.count loop
+      if instr(a_source(i), '/*') > 0 then
+        l_has_ml_comment := true;
+        exit prescan;
+      end if;
+    end loop prescan;
+
+    if not l_has_ml_comment then
+      return a_source;
+    end if;
+
+    <<process_lines>>
+    for i in 1 .. a_source.count loop
+      l_line := a_source(i);
+      -- Fast path: currently inside a multi-line comment
+      if l_in_ml_comment then
+        l_ml_end := instr(l_line, '*/');
+        if l_ml_end > 0 then
+          l_in_ml_comment := false;
+          l_line := substr(l_line, l_ml_end + 2);
+          -- fall through to normal scan of remainder
+        else
+          l_result(i) := '';
+          continue process_lines;
+        end if;
+      end if;
+
+      -- Fast path: no special tokens on this line at all
+      if instr(l_line, '/') = 0
+        and instr(l_line, '-') = 0
+        and instr(l_line, '''') = 0
+      then
+        l_result(i) := l_line;
+        continue process_lines;
+      end if;
+
+      -- Normal scan: consume one token at a time, advance l_remaining
+      l_remaining := l_line;
+      l_line      := null;
+
+      <<scan_line>>
+      loop
+        exit when l_remaining is null or l_remaining = '';
+
+        l_ml_start      := instr(l_remaining, '/*');
+        l_comment_start := instr(l_remaining, '--');
+        l_text_start    := instr(l_remaining, '''');
+        -- only search for q' if ' was found — q' always contains '
+        l_eq_text_start := case when l_text_start > 0
+                            then instr(l_remaining, 'q''')
+                            else 0
+                          end;
+
+        -- count how many tokens are present
+        l_token_count := sign(l_ml_start) + sign(l_comment_start)
+                      + sign(l_text_start) + sign(l_eq_text_start);
+
+        -- no special tokens left — consume remainder and stop
+        if l_token_count = 0 then
+          l_line := l_line || l_remaining;
+          exit scan_line;
+        end if;
+
+        -- only one token present — skip LEAST, use GREATEST to find it
+        if l_token_count = 1 then
+          l_pos := greatest(l_ml_start, l_comment_start, l_text_start, l_eq_text_start);
+        else
+          l_pos := least(
+            case when l_ml_start      > 0 then l_ml_start      else 32767 end,
+            case when l_comment_start > 0 then l_comment_start else 32767 end,
+            case when l_text_start    > 0 then l_text_start    else 32767 end,
+            case when l_eq_text_start > 0 then l_eq_text_start else 32767 end
+          );
+        end if;
+
+        -- q-quoted string: checked before plain quote because q' contains ' and would be misidentified
+        if l_pos = l_eq_text_start
+          and (l_ml_start      = 0 or l_eq_text_start < l_ml_start)
+          and (l_comment_start = 0 or l_eq_text_start < l_comment_start)
+          and (l_text_start    = 0 or l_eq_text_start < l_text_start)
+        then
+          l_eq_end_char := translate(
+            substr(l_remaining, l_eq_text_start + 2, 1),
+            '[{(<', ']})>'
+          );
+          l_end := instr(l_remaining, l_eq_end_char || '''', l_eq_text_start + 3);
+          if l_end > 0 then
+            l_line      := l_line || substr(l_remaining, 1, l_end + 1);
+            l_remaining := substr(l_remaining, l_end + 2);
+          else
+            l_line := l_line || l_remaining;
+            exit scan_line;
+          end if;
+
+        -- Multi-line comment: skip it, continue scanning remainder of line after comment end
+        elsif l_pos = l_ml_start
+              and (l_comment_start = 0 or l_ml_start < l_comment_start)
+              and (l_text_start    = 0 or l_ml_start < l_text_start)
+              and (l_eq_text_start = 0 or l_ml_start < l_eq_text_start)
+        then
+          l_line   := l_line || substr(l_remaining, 1, l_ml_start - 1);
+          l_ml_end := instr(l_remaining, '*/', l_ml_start + 2);
+          if l_ml_end > 0 then
+            l_remaining := substr(l_remaining, l_ml_end + 2);
+          else
+            l_in_ml_comment := true;
+            exit scan_line;
+          end if;
+
+        -- Single-line comment: keep it, stop scanning this line
+        elsif l_pos = l_comment_start
+              and (l_ml_start      = 0 or l_comment_start < l_ml_start)
+              and (l_text_start    = 0 or l_comment_start < l_text_start)
+              and (l_eq_text_start = 0 or l_comment_start < l_eq_text_start)
+        then
+          l_line := l_line || l_remaining;
+          exit scan_line;
+
+        -- Regular string literal: keep it, continue scanning remainder of line after closing quote
+        else
+          -- scan forward continuously to handle '' escaped quotes
+          l_end := l_text_start + 1;
+          loop
+            l_end := instr(l_remaining, '''', l_end);
+            exit when l_end = 0;
+            if substr(l_remaining, l_end, 2) = '''''' then
+              l_end := l_end + 2;  -- skip escaped quote, keep scanning
+            else
+              exit;                -- found real closing quote
+            end if;
+          end loop;
+
+          if l_end > 0 then
+            l_line      := l_line || substr(l_remaining, 1, l_end);
+            l_remaining := substr(l_remaining, l_end + 1);
+          else
+            l_line := l_line || l_remaining;
+            exit scan_line;
+          end if;
+        end if;
+
+      end loop scan_line;
+
+      l_result(i) := l_line;
+
+    end loop process_lines;
+
+    return l_result;
+  end replace_multiline_comments;
+
+
   function replace_multiline_comments(a_source clob) return clob is
     l_result                  clob;
     l_ml_comment_start        binary_integer := 1;

diff --git a/source/core/ut_utils.pks b/source/core/ut_utils.pks
@@ -405,6 +405,9 @@ create or replace package ut_utils authid definer is
   /**
   * Replaces multi-line comments in given source-code with empty lines
   */
+  function replace_multiline_comments(a_source dbms_preprocessor.source_lines_t)
+  return dbms_preprocessor.source_lines_t;
+
   function replace_multiline_comments(a_source clob) return clob;
 
    /**
-Original file line number
+Diff line change
@@ Expand Up @@
        */
       function parse_object_annotations(a_source clob) return ut_annotations;
+      function parse_object_annotations(a_source dbms_preprocessor.source_lines_t) return ut_annotations;
     end;
     /