Read Tokens directly from the Grammar File without invoking JTREE

- read Tokens per REGEX Matcher - move Reserved Keywords from Grammar into ParserKeywordsUtils - adjust the Tests
JSQLParser · wumpz · Oct 25, 2022 · Oct 18, 2021 · Oct 18, 2021 · Oct 23, 2021
commit 83b8a03885b7fe5536e5c18e285ab3f5ca6f9338
diff --git a/src/main/java/net/sf/jsqlparser/parser/ParserKeywordsUtils.java b/src/main/java/net/sf/jsqlparser/parser/ParserKeywordsUtils.java
@@ -16,6 +16,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 public class ParserKeywordsUtils {
@@ -28,20 +29,156 @@ public class ParserKeywordsUtils {
     public final static int RESTRICTED_SQL2016 = 64;
 
     public final static int RESTRICTED_JSQLPARSER =  128
-                                                     | RESTRICTED_FUNCTION
-                                                     | RESTRICTED_SCHEMA
-                                                     | RESTRICTED_TABLE
-                                                     | RESTRICTED_COLUMN
-                                                     | RESTRICTED_EXPRESSION
-                                                     | RESTRICTED_ALIAS
-                                                     | RESTRICTED_SQL2016;
-
-    public static List<String> getDefinedKeywords() throws Exception {
-        return CCJSqlParser.getDefinedKeywords();
-    }
+            | RESTRICTED_FUNCTION
+            | RESTRICTED_SCHEMA
+            | RESTRICTED_TABLE
+            | RESTRICTED_COLUMN
+            | RESTRICTED_EXPRESSION
+            | RESTRICTED_ALIAS
+            | RESTRICTED_SQL2016;
+
+    public static List<String> getReservedKeywords(int restrictriction) {
+        // Classification follows http://www.h2database.com/html/advanced.html#keywords
+        Object[][] ALL_RESERVED_KEYWORDS = {
+                { "ABSENT", RESTRICTED_JSQLPARSER }
+                ,  { "ALL" , RESTRICTED_SQL2016 }
+                ,  { "AND" , RESTRICTED_SQL2016 }
+                ,  { "ANY" , RESTRICTED_JSQLPARSER }
+                ,  { "AS" , RESTRICTED_SQL2016 }
+                ,  { "BETWEEN" , RESTRICTED_SQL2016 }
+                ,  { "BOTH" , RESTRICTED_SQL2016 }
+                ,  { "CASEWHEN" , RESTRICTED_ALIAS }
+                ,  { "CHECK" , RESTRICTED_SQL2016 }
+                ,  { "CONNECT" , RESTRICTED_ALIAS }
+                ,  { "CONNECT_BY_ROOT" , RESTRICTED_JSQLPARSER }
+                ,  { "CONSTRAINT" , RESTRICTED_SQL2016 }
+                ,  { "CREATE" , RESTRICTED_ALIAS }
+                ,  { "CROSS" , RESTRICTED_SQL2016 }
+                ,  { "CURRENT" , RESTRICTED_JSQLPARSER }
+                ,  { "DISTINCT" , RESTRICTED_SQL2016 }
+                ,  { "DOUBLE" , RESTRICTED_ALIAS }
+                ,  { "ELSE" , RESTRICTED_JSQLPARSER }
+                ,  { "EXCEPT" , RESTRICTED_SQL2016 }
+                ,  { "EXISTS" , RESTRICTED_SQL2016 }
+                ,  { "FETCH" , RESTRICTED_SQL2016 }
+                ,  { "FOR" , RESTRICTED_SQL2016 }
+                ,  { "FORCE" , RESTRICTED_SQL2016 }
+                ,  { "FOREIGN" , RESTRICTED_SQL2016 }
+                ,  { "FROM" , RESTRICTED_SQL2016 }
+                ,  { "FULL", RESTRICTED_SQL2016 }
+                ,  { "GROUP", RESTRICTED_SQL2016 }
+                ,  { "GROUPING" , RESTRICTED_ALIAS }
+                ,  { "HAVING" , RESTRICTED_SQL2016 }
+                ,  { "IF" , RESTRICTED_SQL2016 }
+                ,  { "IIF" , RESTRICTED_ALIAS }
+                ,  { "IGNORE" , RESTRICTED_ALIAS }
+                ,  { "ILIKE" , RESTRICTED_SQL2016 }
+                ,  { "IN" , RESTRICTED_SQL2016 }
+                ,  { "INNER" , RESTRICTED_SQL2016 }
+                ,  { "INTERSECT" , RESTRICTED_SQL2016 }
+                ,  { "INTERVAL", RESTRICTED_SQL2016 }
+                ,  { "IS" , RESTRICTED_SQL2016 }
+                ,  { "JOIN" , RESTRICTED_JSQLPARSER }
+                ,  { "LATERAL" , RESTRICTED_SQL2016 }
+                ,  { "LEFT", RESTRICTED_SQL2016 }
+                ,  { "LIKE" , RESTRICTED_SQL2016 }
+                ,  { "LIMIT" , RESTRICTED_SQL2016 }
+                ,  { "MINUS" , RESTRICTED_SQL2016 }
+                ,  { "NATURAL" , RESTRICTED_SQL2016 }
+                ,  { "NOCYCLE" , RESTRICTED_JSQLPARSER }
+                ,  { "NOT", RESTRICTED_SQL2016 }
+                ,  { "NULL" , RESTRICTED_SQL2016 }
+                ,  { "OFFSET" , RESTRICTED_SQL2016 }
+                ,  { "ON" , RESTRICTED_SQL2016 }
+                ,  { "ONLY" , RESTRICTED_JSQLPARSER }
+                ,  { "OPTIMIZE" , RESTRICTED_ALIAS }
+                ,  { "OR" , RESTRICTED_SQL2016 }
+                ,  { "ORDER" , RESTRICTED_SQL2016 }
+                ,  { "OUTER" , RESTRICTED_JSQLPARSER }
+                ,  { "OPTIMIZE ", RESTRICTED_JSQLPARSER }
+                ,  { "PIVOT" , RESTRICTED_JSQLPARSER }
+                ,  { "PROCEDURE" , RESTRICTED_ALIAS }
+                ,  { "PUBLIC", RESTRICTED_ALIAS }
+                ,  { "RECURSIVE" , RESTRICTED_SQL2016 }
+                ,  { "REGEXP" , RESTRICTED_SQL2016 }
+                ,  { "RETURNING" , RESTRICTED_JSQLPARSER }
+                ,  { "RIGHT" , RESTRICTED_SQL2016 }
+                ,  { "SEL" , RESTRICTED_ALIAS }
+                ,  { "SELECT" , RESTRICTED_ALIAS }
+                ,  { "SEMI" , RESTRICTED_JSQLPARSER }
+                ,  { "SET" , RESTRICTED_JSQLPARSER }
+                ,  { "SOME" , RESTRICTED_JSQLPARSER }
+                ,  { "START" , RESTRICTED_JSQLPARSER }
+                ,  { "TABLES" , RESTRICTED_ALIAS }
+                ,  { "TOP" , RESTRICTED_SQL2016 }
+                ,  { "TRAILING", RESTRICTED_SQL2016 }
+                ,  { "UNBOUNDED" , RESTRICTED_JSQLPARSER }
+                ,  { "UNION" , RESTRICTED_SQL2016 }
+                ,  { "UNIQUE" , RESTRICTED_SQL2016 }
+                ,  { "UNPIVOT" , RESTRICTED_JSQLPARSER }
+                ,  { "USE" , RESTRICTED_JSQLPARSER }
+                ,  { "USING" , RESTRICTED_SQL2016 }
+                ,  { "SQL_CACHE" , RESTRICTED_JSQLPARSER }
+                ,  { "SQL_CALC_FOUND_ROWS" , RESTRICTED_JSQLPARSER }
+                ,  { "SQL_NO_CACHE" , RESTRICTED_JSQLPARSER }
+                ,  { "STRAIGHT_JOIN" , RESTRICTED_JSQLPARSER }
+                ,  { "VALUE", RESTRICTED_JSQLPARSER }
+                ,  { "VALUES" , RESTRICTED_SQL2016 }
+                ,  { "VARYING" , RESTRICTED_JSQLPARSER }
+                ,  { "WHEN" , RESTRICTED_SQL2016 }
+                ,  { "WHERE" , RESTRICTED_SQL2016 }
+                ,  { "WINDOW" , RESTRICTED_SQL2016 }
+                ,  { "WITH" , RESTRICTED_SQL2016 }
+                ,  { "XOR", RESTRICTED_JSQLPARSER }
+                ,  { "XMLSERIALIZE" , RESTRICTED_JSQLPARSER }
+
+                // add keywords from the composite token definitions:
+                // tk=<K_DATE_LITERAL> | tk=<K_DATETIMELITERAL> | tk=<K_STRING_FUNCTION_NAME>
+                // we will use the composite tokens instead, which are always hit first before the simple keywords
+                // @todo: figure out a way to remove these composite tokens, as they do more harm than good
+                , { "SEL", RESTRICTED_JSQLPARSER }
+                , { "SELECT", RESTRICTED_JSQLPARSER }
+
+                , { "DATE", RESTRICTED_JSQLPARSER }
+                , { "TIME" , RESTRICTED_JSQLPARSER }
+                , { "TIMESTAMP", RESTRICTED_JSQLPARSER }
+
+                , { "YEAR", RESTRICTED_JSQLPARSER }
+                , { "MONTH", RESTRICTED_JSQLPARSER }
+                , { "DAY", RESTRICTED_JSQLPARSER }
+                , { "HOUR", RESTRICTED_JSQLPARSER }
+                , { "MINUTE" , RESTRICTED_JSQLPARSER }
+                , { "SECOND", RESTRICTED_JSQLPARSER }
+
+                , { "SUBSTR", RESTRICTED_JSQLPARSER }
+                , { "SUBSTRING", RESTRICTED_JSQLPARSER }
+                , { "TRIM", RESTRICTED_JSQLPARSER }
+                , { "POSITION", RESTRICTED_JSQLPARSER }
+                , { "OVERLAY", RESTRICTED_JSQLPARSER }
+
+                , { "NEXTVAL", RESTRICTED_JSQLPARSER }
+
+                //@todo: figure out what those are about
+                , { "RR", RESTRICTED_JSQLPARSER }
+                , { "RS", RESTRICTED_JSQLPARSER }
+                , { "UR", RESTRICTED_JSQLPARSER }
+                , { "CS", RESTRICTED_JSQLPARSER }
+
+                //@todo: Object Names should not start with Hex-Prefix, we shall not find that Token
+                , { "0x", RESTRICTED_JSQLPARSER }
+        };
+
+        ArrayList<String> keywords = new ArrayList<>();
+        for (Object[] data : ALL_RESERVED_KEYWORDS) {
+            int value = (int) data[1];
+
+            // test if bit is not set
+            if ( (value & restrictriction) == restrictriction
+                    || (restrictriction & value) == value )
+                keywords.add( (String) data[0] );
+        }
 
-    public static List<String> getReservedKeywords(int restriction) {
-        return CCJSqlParser.getReservedKeywords(restriction);
+        return keywords;
     }
 
     public static void main(String[] args) throws Exception {
@@ -58,12 +195,34 @@ public static void main(String[] args) throws Exception {
         }
     }
 
-    public static void buildGrammarForRelObjectNameWithoutValue(File file) throws Exception {
-        Pattern pattern = Pattern.compile("String\\W*RelObjectNameWithoutValue\\W*\\(\\W*\\)\\W*:\\s*\\{(?:[^}{]+|\\{(?:[^}{]+|\\{[^}{]*})*})*}\\s*\\{(?:[^}{]+|\\{(?:[^}{]+|\\{[^}{]*})*})*}", Pattern.MULTILINE);
+    public static TreeSet<String> getAllKeywords(File file) throws IOException {
+        Pattern tokenBlockPattern = Pattern.compile("TOKEN\\s*:\\s*(?:/\\*.*\\*/*)\\n\\{(?:[^\\}\\{]+|\\{(?:[^\\}\\{]+|\\{[^\\}\\{]*\\})*\\})*\\}", Pattern.MULTILINE);
+        Pattern tokenStringValuePattern = Pattern.compile("\\\"(\\w{2,})\\\"", Pattern.MULTILINE);
 
         TreeSet<String> allKeywords = new TreeSet<>();
-        allKeywords.addAll(CCJSqlParser.getDefinedKeywords());
-        for (String reserved: CCJSqlParser.getReservedKeywords(RESTRICTED_JSQLPARSER)) {
+
+        Path path = file.toPath();
+        Charset charset = Charset.defaultCharset();
+        String content = new String(Files.readAllBytes(path), charset);
+
+        Matcher tokenBlockmatcher = tokenBlockPattern.matcher(content);
+        while (tokenBlockmatcher.find()) {
+            String tokenBlock = tokenBlockmatcher.group(0);
+            Matcher tokenStringValueMatcher= tokenStringValuePattern.matcher(tokenBlock);
+            while (tokenStringValueMatcher.find()) {
+                String tokenValue=tokenStringValueMatcher.group(1);
+                allKeywords.add(tokenValue);
+            }
+        }
+        return allKeywords;
+    }
+
+    public static void buildGrammarForRelObjectNameWithoutValue(File file) throws Exception {
+        Pattern methodBlockPattern = Pattern.compile("String\\W*RelObjectNameWithoutValue\\W*\\(\\W*\\)\\W*:\\s*\\{(?:[^}{]+|\\{(?:[^}{]+|\\{[^}{]*})*})*}\\s*\\{(?:[^}{]+|\\{(?:[^}{]+|\\{[^}{]*})*})*}", Pattern.MULTILINE);
+
+        TreeSet<String> allKeywords = getAllKeywords(file);
+
+        for (String reserved: getReservedKeywords(RESTRICTED_JSQLPARSER)) {
             allKeywords.remove(reserved);
         }
 
@@ -82,17 +241,17 @@ public static void buildGrammarForRelObjectNameWithoutValue(File file) throws Ex
                        + "    { return tk.image; }\n"
                        + "}");
 
-        replaceInFile(file, pattern, builder.toString());
+        replaceInFile(file, methodBlockPattern, builder.toString());
     }
 
     public static void buildGrammarForRelObjectName(File file) throws Exception {
         // Pattern pattern = Pattern.compile("String\\W*RelObjectName\\W*\\(\\W*\\)\\W*:\\s*\\{(?:[^}{]+|\\{(?:[^}{]+|\\{[^}{]*})*})*}\\s*\\{(?:[^}{]+|\\{(?:[^}{]+|\\{[^}{]*})*})*}", Pattern.MULTILINE);
         TreeSet<String> allKeywords = new TreeSet<>();
-        for (String reserved: CCJSqlParser.getReservedKeywords(RESTRICTED_ALIAS)) {
+        for (String reserved: getReservedKeywords(RESTRICTED_ALIAS)) {
             allKeywords.add(reserved);
         }
 
-        for (String reserved: CCJSqlParser.getReservedKeywords(RESTRICTED_JSQLPARSER & ~RESTRICTED_ALIAS)) {
+        for (String reserved: getReservedKeywords(RESTRICTED_JSQLPARSER & ~RESTRICTED_ALIAS)) {
             allKeywords.remove(reserved);
         }