Add Vim NFA regexp engine dump log helpers

guns · guns · commit 742216c76650 · 2013-08-01T20:31:10.000-05:00
diff --git a/clj/src/vim_clojure_static/generate.clj b/clj/src/vim_clojure_static/generate.clj
@@ -2,8 +2,9 @@
 ;;          Joel Holdbrooks <cjholdbrooks@gmail.com>
 
 (ns vim-clojure-static.generate
-  (:require [clojure.string :as string]
-            [clojure.set :as set]))
+  (:require [clojure.java.shell :as shell]
+            [clojure.set :as set]
+            [clojure.string :as string]))
 
 ;;
 ;; Helpers
@@ -221,7 +222,18 @@
                            (fmt "script=" :script)
                            (fmt "block=" :block)])))
 
+(defn vim-nfa-dump
+  "Run a patched version of Vim compiled with -DDEBUG on a new file containing
+   buffer, then move the NFA log to log-path. The patch is located at
+   vim/custom-nfa-log.patch"
+  [vim-path buffer log-path]
+  (let [file "tmp/nfa-test-file.clj"]
+    (spit file buffer)
+    (shell/sh vim-path "-u" "NONE" "-N" "-S" "vim/test-runtime.vim" file)
+    (shell/sh "mv" "nfa_regexp.log" log-path)))
+
 (comment
+  ;; Generate the vim literal definitions for pasting into the runtime files.
   (spit "tmp/clojure-defs.vim"
         (str generation-comment
              clojure-version-comment
@@ -239,6 +251,23 @@
              vim-unicode-category-char-classes
              vim-unicode-script-char-classes
              vim-unicode-block-char-classes))
-  (spit "tmp/clojure-regexp-properties.clj"
+  (spit "tmp/nfa-test-file.clj"
         comprehensive-clojure-character-property-regexps)
+  ;; Create regexp engine dump files and compare number of operations.
+  (let [syn-path "../syntax/clojure.vim"
+        orig-syn-buf (slurp syn-path)
+        vim-path (or (System/getenv "VIMDEBUG") "vim")
+        buf (format "#\"\\p{%s}\"\n" "Ll")
+        buf-hash (hash buf)]
+    (try
+      (mapv (fn [path]
+              (let [syn-buf (slurp path)
+                    syn-hash (hash syn-buf)
+                    log-path (format "tmp/debug:%d:%d.log" buf-hash syn-hash)]
+                (spit syn-path syn-buf)
+                (vim-nfa-dump vim-path buf log-path)
+                (count (re-seq #"\n" (slurp log-path)))))
+            ["../syntax/clojure.vim" "tmp/altsyntax.vim"])
+      (finally
+        (spit syn-path orig-syn-buf))))
   )
diff --git a/clj/src/vim_clojure_static/test.clj b/clj/src/vim_clojure_static/test.clj
@@ -17,7 +17,7 @@
   [file & lines]
   (io/make-parents file)
   (spit file (string/join \newline lines))
-  (shell/sh "vim" "-u" "NONE" "-N" "-S" "vim/syn-id-names.vim" file)
+  (shell/sh "vim" "-u" "NONE" "-N" "-S" "vim/test-runtime.vim" file)
   ;; The last line of the file will contain valid EDN
   (into {} (map (fn [l ids] [l (mapv keyword ids)])
                 lines
diff --git a/clj/vim/custom-nfa-log.patch b/clj/vim/custom-nfa-log.patch
@@ -0,0 +1,75 @@
+commit 4f385fb359c5d51bd4a079ffd35bbcc1c6676c4a (HEAD, github/custom-nfa-log, custom-nfa-log)
+Author: guns <self@sungpae.com>
+Date:   9 hours ago
+
+    Add custom NFA logging to nfa_regexp.log
+    
+    The goal is to get a ballpark estimate of the number of steps the regexp
+    engine undertakes in a session. One line of logging is done:
+    
+    * Start of nfa_regmatch()
+    * For each character considered
+    * For each state of a character considered
+    
+    Vimm should be compiled with -DDEBUG; if the NFA engine state graphs are
+    desired, then also use -DINCLUDE_NFA_DUMP.
+    
+    This branch can also be found at:
+    
+    https://github.com/guns/vim/tree/custom-nfa-log
+---
+ src/regexp_nfa.c | 20 +++++++++++++++++---
+ 1 file changed, 17 insertions(+), 3 deletions(-)
+
+diff --git a/src/regexp_nfa.c b/src/regexp_nfa.c
+index 738ac3b..c39f94d 100644
+--- a/src/regexp_nfa.c
++++ b/src/regexp_nfa.c
+@@ -24,9 +24,18 @@
+ #ifdef DEBUG
+ # define NFA_REGEXP_ERROR_LOG	"nfa_regexp_error.log"
+ # define ENABLE_LOG
+-# define NFA_REGEXP_DUMP_LOG	"nfa_regexp_dump.log"
+-# define NFA_REGEXP_RUN_LOG	"nfa_regexp_run.log"
+-# define NFA_REGEXP_DEBUG_LOG	"nfa_regexp_debug.log"
++# ifdef INCLUDE_NFA_DUMP
++#  define NFA_REGEXP_DUMP_LOG	"/dev/null"
++# else
++#  define NFA_REGEXP_DUMP_LOG	"nfa_regexp.log"
++# endif
++# define NFA_REGEXP_RUN_LOG	"/dev/null"
++# define NFA_REGEXP_DEBUG_LOG	"/dev/null"
++# define LOG(fmt, ...) do {			\
++    FILE *log = fopen("nfa_regexp.log", "a");	\
++    fprintf(log, fmt, __VA_ARGS__);		\
++    fclose(log);				\
++} while (0)
+ #endif
+ 
+ enum
+@@ -5043,6 +5052,8 @@ nfa_regmatch(prog, start, submatch, m)
+ 	goto theend;
+ 
+ #ifdef ENABLE_LOG
++    LOG("START nfa_regmatch: alloc=%d pattern=\"%s\"\n", size*2, prog->pattern);
++
+     log_fd = fopen(NFA_REGEXP_RUN_LOG, "a");
+     if (log_fd != NULL)
+     {
+@@ -5129,6 +5140,8 @@ nfa_regmatch(prog, start, submatch, m)
+ 	nextlist->id = nfa_listid + 1;
+ 
+ #ifdef ENABLE_LOG
++	LOG("# states=%d reginput=\"%s\"\n", thislist->n, reginput);
++
+ 	fprintf(log_fd, "------------------------------------------\n");
+ 	fprintf(log_fd, ">>> Reginput is \"%s\"\n", reginput);
+ 	fprintf(log_fd, ">>> Advanced one character ... Current char is %c (code %d) \n", curc, (int)curc);
+@@ -5161,6 +5174,7 @@ nfa_regmatch(prog, start, submatch, m)
+ 	    fprintf(debug, "%s, ", code);
+ #endif
+ #ifdef ENABLE_LOG
++	    LOG("## computing nextlist: code=\"%s\"\n", code);
+ 	    {
+ 		int col;
+ 
diff --git a/clj/vim/test-runtime.vim b/clj/vim/test-runtime.vim
@@ -3,16 +3,23 @@
 execute 'set rtp=' . expand('%:p:h:h:h') . ',$VIMRUNTIME'
 filetype plugin on
 syntax on
+set synmaxcol=0
 setfiletype clojure
 
-function! s:append_syn_id_names()
+if !exists('g:testing')
+    let g:testing = 1
+endif
+
+function! s:syn_id_names()
     let names = []
     for lnum in range(1, line('$'))
         let f = 'synIDattr(synID(' . lnum . ', v:val, 0), "name")'
         call add(names, map(range(1, virtcol([lnum, '$']) - 1), f))
     endfor
-    " Changing the quotes will make this valid EDN
-    call append(line('$'), tr(string(names), "'", '"'))
+    return names
 endfunction
 
-call s:append_syn_id_names() | write | quitall!
+if g:testing
+    " Changing the quotes will make this valid EDN
+    call append(line('$'), tr(string(s:syn_id_names()), "'", '"')) | write | quitall!
+endif
