// Licensed to the Apache Software Foundation (ASF) under one

// or more contributor license agreements. See the NOTICE file

// distributed with this work for additional information

// regarding copyright ownership. The ASF licenses this file

// to you under the Apache License, Version 2.0 (the

// "License"); you may not use this file except in compliance

// with the License. You may obtain a copy of the License at

//

// http://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing,

// software distributed under the License is distributed on an

// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

// KIND, either express or implied. See the License for the

// specific language governing permissions and limitations

// under the License.

// This file is copied from

// https://github.com/cloudera/Impala/blob/v0.7refresh/fe/src/main/jflex/sql-scanner.flex

// and modified by Doris

package org.apache.doris.analysis;

import java_cup.runtime.Symbol;

import java.io.StringWriter;

import java.lang.Integer;

import java.math.BigDecimal;

import java.math.BigInteger;

import java.util.HashMap;

import java.util.LinkedHashMap;

import java.util.Map;

import java.util.Iterator;

import java.util.List;

import java.util.ArrayList;

import org.apache.doris.analysis.SqlParserSymbols;

import org.apache.doris.common.util.SqlUtils;

import org.apache.doris.qe.SqlModeHelper;

%%

%class SqlScanner

%cup

%public

%final

%eofval{

return newToken(SqlParserSymbols.EOF, null);

%eofval}

%unicode

%line

%column

%ctorarg Long sql_mode

%{

// Help to judge a integer-literal is bigger than LARGEINT_MAX

// NOTE: the 'longMin' is not '-2^63' here, to make sure the return value functions

// like 'long abs(long)' is same with the type of arguments, which is valid.

private static final BigInteger LONG_MAX = new BigInteger("9223372036854775807"); // 2^63 - 1

private static final BigInteger LARGEINT_MAX_ABS = new BigInteger("170141183460469231731687303715884105728"); // 2^127

// This param will affect the tokens returned by scanner.

// For example:

// In PIPES_AS_CONCAT_MODE(0x0002), scanner will return token with id as KW_PIPE instead of KW_OR when '||' is scanned.

private long sql_mode;

/**

* Creates a new scanner to chain-call the generated constructor.

* There is also a java.io.InputStream version of this constructor.

* If you use this constructor, sql_mode will be set to 0 (default)

*

* @param in the java.io.Reader to read input from.

*/

public SqlScanner(java.io.Reader in) {

this(in, 0L);

}

/**

* Creates a new scanner chain-call the generated constructor.

* There is also java.io.Reader version of this constructor.

* If you use this constructor, sql_mode will be set to 0 (default)

*

* @param in the java.io.Inputstream to read input from.

*/

public SqlScanner(java.io.InputStream in) {

this(in, 0L);

}

// map from keyword string to token id

// we use a linked hash map because the insertion order is important.

// for example, we want "and" to come after "&&" to make sure error reporting

// uses "and" as a display name and not "&&"

private static final Map<String, Integer> keywordMap = new LinkedHashMap<String, Integer>();

static {

keywordMap.put("&&", new Integer(SqlParserSymbols.KW_AND));

keywordMap.put("||", new Integer(SqlParserSymbols.KW_PIPE));

keywordMap.put("account_lock", new Integer(SqlParserSymbols.KW_ACCOUNT_LOCK));

keywordMap.put("account_unlock", new Integer(SqlParserSymbols.KW_ACCOUNT_UNLOCK));

keywordMap.put("add", new Integer(SqlParserSymbols.KW_ADD));

keywordMap.put("admin", new Integer(SqlParserSymbols.KW_ADMIN));

keywordMap.put("after", new Integer(SqlParserSymbols.KW_AFTER));

keywordMap.put("aggregate", new Integer(SqlParserSymbols.KW_AGGREGATE));

keywordMap.put("alias", new Integer(SqlParserSymbols.KW_ALIAS));

keywordMap.put("all", new Integer(SqlParserSymbols.KW_ALL));

keywordMap.put("alter", new Integer(SqlParserSymbols.KW_ALTER));

keywordMap.put("always", new Integer(SqlParserSymbols.KW_ALWAYS));

keywordMap.put("and", new Integer(SqlParserSymbols.KW_AND));

keywordMap.put("analyze", new Integer(SqlParserSymbols.KW_ANALYZE));

keywordMap.put("anti", new Integer(SqlParserSymbols.KW_ANTI));

keywordMap.put("append", new Integer(SqlParserSymbols.KW_APPEND));

keywordMap.put("array", new Integer(SqlParserSymbols.KW_ARRAY));

keywordMap.put("auto_increment", new Integer(SqlParserSymbols.KW_AUTO_INCREMENT));

keywordMap.put("as", new Integer(SqlParserSymbols.KW_AS));

keywordMap.put("asc", new Integer(SqlParserSymbols.KW_ASC));

keywordMap.put("at", new Integer(SqlParserSymbols.KW_AT));

keywordMap.put("authors", new Integer(SqlParserSymbols.KW_AUTHORS));

keywordMap.put("backend", new Integer(SqlParserSymbols.KW_BACKEND));

keywordMap.put("backends", new Integer(SqlParserSymbols.KW_BACKENDS));

keywordMap.put("backup", new Integer(SqlParserSymbols.KW_BACKUP));

keywordMap.put("begin", new Integer(SqlParserSymbols.KW_BEGIN));

keywordMap.put("belong", new Integer(SqlParserSymbols.KW_BELONG));

keywordMap.put("between", new Integer(SqlParserSymbols.KW_BETWEEN));

keywordMap.put("bigint", new Integer(SqlParserSymbols.KW_BIGINT));

keywordMap.put("bin", new Integer(SqlParserSymbols.KW_BIN));

keywordMap.put("binlog", new Integer(SqlParserSymbols.KW_BINLOG));

keywordMap.put("bitmap", new Integer(SqlParserSymbols.KW_BITMAP));

keywordMap.put("inverted", new Integer(SqlParserSymbols.KW_INVERTED));

keywordMap.put("bitmap_empty", new Integer(SqlParserSymbols.KW_BITMAP_EMPTY));

keywordMap.put("bitmap_union", new Integer(SqlParserSymbols.KW_BITMAP_UNION));

keywordMap.put("ngram_bf", new Integer(SqlParserSymbols.KW_NGRAM_BF));

keywordMap.put("blob", new Integer(SqlParserSymbols.KW_BLOB));

keywordMap.put("boolean", new Integer(SqlParserSymbols.KW_BOOLEAN));

keywordMap.put("brief", new Integer(SqlParserSymbols.KW_BRIEF));

keywordMap.put("broker", new Integer(SqlParserSymbols.KW_BROKER));

keywordMap.put("buckets", new Integer(SqlParserSymbols.KW_BUCKETS));

keywordMap.put("build", new Integer(SqlParserSymbols.KW_BUILD));

keywordMap.put("builtin", new Integer(SqlParserSymbols.KW_BUILTIN));

keywordMap.put("by", new Integer(SqlParserSymbols.KW_BY));

keywordMap.put("cached", new Integer(SqlParserSymbols.KW_CACHED));

keywordMap.put("cancel", new Integer(SqlParserSymbols.KW_CANCEL));

keywordMap.put("cache", new Integer(SqlParserSymbols.KW_CACHE));

keywordMap.put("case", new Integer(SqlParserSymbols.KW_CASE));

keywordMap.put("cast", new Integer(SqlParserSymbols.KW_CAST));

keywordMap.put("catalog", new Integer(SqlParserSymbols.KW_CATALOG));

keywordMap.put("catalogs", new Integer(SqlParserSymbols.KW_CATALOGS));

keywordMap.put("chain", new Integer(SqlParserSymbols.KW_CHAIN));

keywordMap.put("char", new Integer(SqlParserSymbols.KW_CHAR));

keywordMap.put("character", new Integer(SqlParserSymbols.KW_CHAR));

keywordMap.put("charset", new Integer(SqlParserSymbols.KW_CHARSET));

keywordMap.put("check", new Integer(SqlParserSymbols.KW_CHECK));

keywordMap.put("clean", new Integer(SqlParserSymbols.KW_CLEAN));

keywordMap.put("cluster", new Integer(SqlParserSymbols.KW_CLUSTER));

keywordMap.put("clusters", new Integer(SqlParserSymbols.KW_CLUSTERS));

keywordMap.put("collate", new Integer(SqlParserSymbols.KW_COLLATE));

keywordMap.put("collation", new Integer(SqlParserSymbols.KW_COLLATION));

keywordMap.put("colocate", new Integer(SqlParserSymbols.KW_COLOCATE));

keywordMap.put("column", new Integer(SqlParserSymbols.KW_COLUMN));

keywordMap.put("columns", new Integer(SqlParserSymbols.KW_COLUMNS));

keywordMap.put("comment", new Integer(SqlParserSymbols.KW_COMMENT));

keywordMap.put("commit", new Integer(SqlParserSymbols.KW_COMMIT));

keywordMap.put("committed", new Integer(SqlParserSymbols.KW_COMMITTED));

keywordMap.put("compact", new Integer(SqlParserSymbols.KW_COMPACT));

keywordMap.put("complete", new Integer(SqlParserSymbols.KW_COMPLETE));

keywordMap.put("compute", new Integer(SqlParserSymbols.KW_COMPUTE));

keywordMap.put("config", new Integer(SqlParserSymbols.KW_CONFIG));

keywordMap.put("connection", new Integer(SqlParserSymbols.KW_CONNECTION));

keywordMap.put("connection_id", new Integer(SqlParserSymbols.KW_CONNECTION_ID));

keywordMap.put("consistent", new Integer(SqlParserSymbols.KW_CONSISTENT));

keywordMap.put("convert", new Integer(SqlParserSymbols.KW_CONVERT));

keywordMap.put("copy", new Integer(SqlParserSymbols.KW_COPY));

keywordMap.put("count", new Integer(SqlParserSymbols.KW_COUNT));

keywordMap.put("create", new Integer(SqlParserSymbols.KW_CREATE));

keywordMap.put("creation", new Integer(SqlParserSymbols.KW_CREATION));

keywordMap.put("cross", new Integer(SqlParserSymbols.KW_CROSS));

keywordMap.put("cube", new Integer(SqlParserSymbols.KW_CUBE));

keywordMap.put("current", new Integer(SqlParserSymbols.KW_CURRENT));

keywordMap.put("current_catalog", new Integer(SqlParserSymbols.KW_CURRENT_CATALOG));

keywordMap.put("current_timestamp", new Integer(SqlParserSymbols.KW_CURRENT_TIMESTAMP));

keywordMap.put("current_user", new Integer(SqlParserSymbols.KW_CURRENT_USER));

keywordMap.put("data", new Integer(SqlParserSymbols.KW_DATA));

keywordMap.put("database", new Integer(SqlParserSymbols.KW_DATABASE));

keywordMap.put("databases", new Integer(SqlParserSymbols.KW_DATABASES));

keywordMap.put("date", new Integer(SqlParserSymbols.KW_DATE));

keywordMap.put("datev1", new Integer(SqlParserSymbols.KW_DATEV1));

keywordMap.put("datev2", new Integer(SqlParserSymbols.KW_DATEV2));

keywordMap.put("datetime", new Integer(SqlParserSymbols.KW_DATETIME));

keywordMap.put("datetimev1", new Integer(SqlParserSymbols.KW_DATETIMEV1));

keywordMap.put("datetimev2", new Integer(SqlParserSymbols.KW_DATETIMEV2));

keywordMap.put("time", new Integer(SqlParserSymbols.KW_TIME));

keywordMap.put("day", new Integer(SqlParserSymbols.KW_DAY));

keywordMap.put("decimal", new Integer(SqlParserSymbols.KW_DECIMAL));

keywordMap.put("decimalv2", new Integer(SqlParserSymbols.KW_DECIMALV2));

keywordMap.put("decimalv3", new Integer(SqlParserSymbols.KW_DECIMALV3));

keywordMap.put("decommission", new Integer(SqlParserSymbols.KW_DECOMMISSION));

keywordMap.put("default", new Integer(SqlParserSymbols.KW_DEFAULT));

keywordMap.put("deferred", new Integer(SqlParserSymbols.KW_DEFERRED));

keywordMap.put("delete", new Integer(SqlParserSymbols.KW_DELETE));

keywordMap.put("demand", new Integer(SqlParserSymbols.KW_DEMAND));

keywordMap.put("desc", new Integer(SqlParserSymbols.KW_DESC));

keywordMap.put("describe", new Integer(SqlParserSymbols.KW_DESCRIBE));

keywordMap.put("diagnose", new Integer(SqlParserSymbols.KW_DIAGNOSE));

keywordMap.put("diagnosis", new Integer(SqlParserSymbols.KW_DIAGNOSIS));

keywordMap.put("disk", new Integer(SqlParserSymbols.KW_DISK));

keywordMap.put("distinct", new Integer(SqlParserSymbols.KW_DISTINCT));

keywordMap.put("distinctpc", new Integer(SqlParserSymbols.KW_DISTINCTPC));

keywordMap.put("distinctpcsa", new Integer(SqlParserSymbols.KW_DISTINCTPCSA));

keywordMap.put("distributed", new Integer(SqlParserSymbols.KW_DISTRIBUTED));

keywordMap.put("distribution", new Integer(SqlParserSymbols.KW_DISTRIBUTION));

keywordMap.put("div", new Integer(SqlParserSymbols.KW_DIV));

keywordMap.put("do", new Integer(SqlParserSymbols.KW_DO));

keywordMap.put("doris_internal_table_id", new Integer(SqlParserSymbols.KW_DORIS_INTERNAL_TABLE_ID));

keywordMap.put("double", new Integer(SqlParserSymbols.KW_DOUBLE));

keywordMap.put("drop", new Integer(SqlParserSymbols.KW_DROP));

keywordMap.put("dropp", new Integer(SqlParserSymbols.KW_DROPP));

keywordMap.put("dual", new Integer(SqlParserSymbols.KW_DUAL));

keywordMap.put("duplicate", new Integer(SqlParserSymbols.KW_DUPLICATE));

keywordMap.put("dynamic", new Integer(SqlParserSymbols.KW_DYNAMIC));

keywordMap.put("else", new Integer(SqlParserSymbols.KW_ELSE));

keywordMap.put("enable", new Integer(SqlParserSymbols.KW_ENABLE));

keywordMap.put("encryptkey", new Integer(SqlParserSymbols.KW_ENCRYPTKEY));

keywordMap.put("encryptkeys", new Integer(SqlParserSymbols.KW_ENCRYPTKEYS));

keywordMap.put("end", new Integer(SqlParserSymbols.KW_END));

keywordMap.put("ends", new Integer(SqlParserSymbols.KW_ENDS));

keywordMap.put("engine", new Integer(SqlParserSymbols.KW_ENGINE));

keywordMap.put("engines", new Integer(SqlParserSymbols.KW_ENGINES));

keywordMap.put("enter", new Integer(SqlParserSymbols.KW_ENTER));

keywordMap.put("errors", new Integer(SqlParserSymbols.KW_ERRORS));

keywordMap.put("events", new Integer(SqlParserSymbols.KW_EVENTS));

keywordMap.put("every", new Integer(SqlParserSymbols.KW_EVERY));

keywordMap.put("except", new Integer(SqlParserSymbols.KW_EXCEPT));

keywordMap.put("exclude", new Integer(SqlParserSymbols.KW_EXCLUDE));

keywordMap.put("exists", new Integer(SqlParserSymbols.KW_EXISTS));

keywordMap.put("explain", new Integer(SqlParserSymbols.KW_DESCRIBE));

keywordMap.put("export", new Integer(SqlParserSymbols.KW_EXPORT));

keywordMap.put("extended", new Integer(SqlParserSymbols.KW_EXTENDED));

keywordMap.put("external", new Integer(SqlParserSymbols.KW_EXTERNAL));

keywordMap.put("extract", new Integer(SqlParserSymbols.KW_EXTRACT));

keywordMap.put("false", new Integer(SqlParserSymbols.KW_FALSE));

keywordMap.put("fast", new Integer(SqlParserSymbols.KW_FAST));

keywordMap.put("failed_login_attempts", new Integer(SqlParserSymbols.KW_FAILED_LOGIN_ATTEMPTS));

keywordMap.put("feature", new Integer(SqlParserSymbols.KW_FEATURE));

keywordMap.put("fields", new Integer(SqlParserSymbols.KW_FIELDS));

keywordMap.put("file", new Integer(SqlParserSymbols.KW_FILE));

keywordMap.put("filter", new Integer(SqlParserSymbols.KW_FILTER));

keywordMap.put("first", new Integer(SqlParserSymbols.KW_FIRST));

keywordMap.put("float", new Integer(SqlParserSymbols.KW_FLOAT));

keywordMap.put("follower", new Integer(SqlParserSymbols.KW_FOLLOWER));

keywordMap.put("following", new Integer(SqlParserSymbols.KW_FOLLOWING));

keywordMap.put("for", new Integer(SqlParserSymbols.KW_FOR));

keywordMap.put("force", new Integer(SqlParserSymbols.KW_FORCE));

keywordMap.put("format", new Integer(SqlParserSymbols.KW_FORMAT));

keywordMap.put("compress_type", new Integer(SqlParserSymbols.KW_COMPRESS_TYPE));

keywordMap.put("free", new Integer(SqlParserSymbols.KW_FREE));

keywordMap.put("from", new Integer(SqlParserSymbols.KW_FROM));

keywordMap.put("frontend", new Integer(SqlParserSymbols.KW_FRONTEND));

keywordMap.put("frontends", new Integer(SqlParserSymbols.KW_FRONTENDS));

keywordMap.put("full", new Integer(SqlParserSymbols.KW_FULL));

keywordMap.put("function", new Integer(SqlParserSymbols.KW_FUNCTION));

keywordMap.put("functions", new Integer(SqlParserSymbols.KW_FUNCTIONS));

keywordMap.put("type_cast", new Integer(SqlParserSymbols.KW_TYPECAST));

keywordMap.put("generated", new Integer(SqlParserSymbols.KW_GENERATED));

keywordMap.put("generic", new Integer(SqlParserSymbols.KW_GENERIC));

keywordMap.put("global", new Integer(SqlParserSymbols.KW_GLOBAL));

keywordMap.put("grant", new Integer(SqlParserSymbols.KW_GRANT));

keywordMap.put("grants", new Integer(SqlParserSymbols.KW_GRANTS));

keywordMap.put("graph", new Integer(SqlParserSymbols.KW_GRAPH));

keywordMap.put("group", new Integer(SqlParserSymbols.KW_GROUP));

keywordMap.put("grouping", new Integer(SqlParserSymbols.KW_GROUPING));

keywordMap.put("groups", new Integer(SqlParserSymbols.KW_GROUPS));

keywordMap.put("hash", new Integer(SqlParserSymbols.KW_HASH));

keywordMap.put("having", new Integer(SqlParserSymbols.KW_HAVING));

keywordMap.put("hdfs", new Integer(SqlParserSymbols.KW_HDFS));

keywordMap.put("help", new Integer(SqlParserSymbols.KW_HELP));

keywordMap.put("hll", new Integer(SqlParserSymbols.KW_HLL));

keywordMap.put("hll_union", new Integer(SqlParserSymbols.KW_HLL_UNION));

keywordMap.put("hostname", new Integer(SqlParserSymbols.KW_HOSTNAME));

keywordMap.put("hour", new Integer(SqlParserSymbols.KW_HOUR));

keywordMap.put("hotspot", new Integer(SqlParserSymbols.KW_HOTSPOT));

keywordMap.put("hub", new Integer(SqlParserSymbols.KW_HUB));

keywordMap.put("identified", new Integer(SqlParserSymbols.KW_IDENTIFIED));

keywordMap.put("if", new Integer(SqlParserSymbols.KW_IF));

keywordMap.put("immediate", new Integer(SqlParserSymbols.KW_IMMEDIATE));

keywordMap.put("in", new Integer(SqlParserSymbols.KW_IN));

keywordMap.put("incremental", new Integer(SqlParserSymbols.KW_INCREMENTAL));

keywordMap.put("index", new Integer(SqlParserSymbols.KW_INDEX));

keywordMap.put("indexes", new Integer(SqlParserSymbols.KW_INDEXES));

keywordMap.put("infile", new Integer(SqlParserSymbols.KW_INFILE));

keywordMap.put("inner", new Integer(SqlParserSymbols.KW_INNER));

keywordMap.put("insert", new Integer(SqlParserSymbols.KW_INSERT));

keywordMap.put("install", new Integer(SqlParserSymbols.KW_INSTALL));

keywordMap.put("int", new Integer(SqlParserSymbols.KW_INT));

keywordMap.put("integer", new Integer(SqlParserSymbols.KW_INT));

keywordMap.put("intermediate", new Integer(SqlParserSymbols.KW_INTERMEDIATE));

keywordMap.put("intersect", new Integer(SqlParserSymbols.KW_INTERSECT));

keywordMap.put("interval", new Integer(SqlParserSymbols.KW_INTERVAL));

keywordMap.put("into", new Integer(SqlParserSymbols.KW_INTO));

keywordMap.put("ipv4", new Integer(SqlParserSymbols.KW_IPV4));

keywordMap.put("ipv6", new Integer(SqlParserSymbols.KW_IPV6));

keywordMap.put("is", new Integer(SqlParserSymbols.KW_IS));

keywordMap.put("isnull", new Integer(SqlParserSymbols.KW_ISNULL));

keywordMap.put("isolation", new Integer(SqlParserSymbols.KW_ISOLATION));

keywordMap.put("job", new Integer(SqlParserSymbols.KW_JOB));

keywordMap.put("jobs", new Integer(SqlParserSymbols.KW_JOBS));

keywordMap.put("join", new Integer(SqlParserSymbols.KW_JOIN));

keywordMap.put("json", new Integer(SqlParserSymbols.KW_JSON));

keywordMap.put("jsonb", new Integer(SqlParserSymbols.KW_JSONB));

keywordMap.put("variant", new Integer(SqlParserSymbols.KW_VARIANT));

keywordMap.put("key", new Integer(SqlParserSymbols.KW_KEY));

keywordMap.put("keys", new Integer(SqlParserSymbols.KW_KEYS));

keywordMap.put("kill", new Integer(SqlParserSymbols.KW_KILL));

keywordMap.put("label", new Integer(SqlParserSymbols.KW_LABEL));

keywordMap.put("largeint", new Integer(SqlParserSymbols.KW_LARGEINT));

keywordMap.put("last", new Integer(SqlParserSymbols.KW_LAST));

keywordMap.put("lateral", new Integer(SqlParserSymbols.KW_LATERAL));

keywordMap.put("ldap", new Integer(SqlParserSymbols.KW_LDAP));

keywordMap.put("ldap_admin_password", new Integer(SqlParserSymbols.KW_LDAP_ADMIN_PASSWORD));

keywordMap.put("left", new Integer(SqlParserSymbols.KW_LEFT));

keywordMap.put("less", new Integer(SqlParserSymbols.KW_LESS));

keywordMap.put("level", new Integer(SqlParserSymbols.KW_LEVEL));

keywordMap.put("like", new Integer(SqlParserSymbols.KW_LIKE));

keywordMap.put("limit", new Integer(SqlParserSymbols.KW_LIMIT));

keywordMap.put("link", new Integer(SqlParserSymbols.KW_LINK));

keywordMap.put("list", new Integer(SqlParserSymbols.KW_LIST));

keywordMap.put("load", new Integer(SqlParserSymbols.KW_LOAD));

keywordMap.put("local", new Integer(SqlParserSymbols.KW_LOCAL));

keywordMap.put("location", new Integer(SqlParserSymbols.KW_LOCATION));

keywordMap.put("lock", new Integer(SqlParserSymbols.KW_LOCK));

keywordMap.put("low_priority", new Integer(SqlParserSymbols.KW_LOW_PRIORITY));

keywordMap.put("map", new Integer(SqlParserSymbols.KW_MAP));

keywordMap.put("match", new Integer(SqlParserSymbols.KW_MATCH));

keywordMap.put("match_any", new Integer(SqlParserSymbols.KW_MATCH_ANY));

keywordMap.put("match_all", new Integer(SqlParserSymbols.KW_MATCH_ALL));

keywordMap.put("match_phrase", new Integer(SqlParserSymbols.KW_MATCH_PHRASE));

keywordMap.put("match_phrase_prefix", new Integer(SqlParserSymbols.KW_MATCH_PHRASE_PREFIX));

keywordMap.put("match_regexp", new Integer(SqlParserSymbols.KW_MATCH_REGEXP));

keywordMap.put("match_phrase_edge", new Integer(SqlParserSymbols.KW_MATCH_PHRASE_EDGE));

keywordMap.put("materialized", new Integer(SqlParserSymbols.KW_MATERIALIZED));

keywordMap.put("max", new Integer(SqlParserSymbols.KW_MAX));

keywordMap.put("maxvalue", new Integer(SqlParserSymbols.KW_MAX_VALUE));

keywordMap.put("merge", new Integer(SqlParserSymbols.KW_MERGE));

keywordMap.put("migrate", new Integer(SqlParserSymbols.KW_MIGRATE));

keywordMap.put("migrations", new Integer(SqlParserSymbols.KW_MIGRATIONS));

keywordMap.put("min", new Integer(SqlParserSymbols.KW_MIN));

keywordMap.put("minus", new Integer(SqlParserSymbols.KW_MINUS));

keywordMap.put("minute", new Integer(SqlParserSymbols.KW_MINUTE));

keywordMap.put("modify", new Integer(SqlParserSymbols.KW_MODIFY));

keywordMap.put("month", new Integer(SqlParserSymbols.KW_MONTH));

keywordMap.put("name", new Integer(SqlParserSymbols.KW_NAME));

keywordMap.put("names", new Integer(SqlParserSymbols.KW_NAMES));

keywordMap.put("natural", new Integer(SqlParserSymbols.KW_NATURAL));

keywordMap.put("negative", new Integer(SqlParserSymbols.KW_NEGATIVE));

keywordMap.put("never", new Integer(SqlParserSymbols.KW_NEVER));

keywordMap.put("next", new Integer(SqlParserSymbols.KW_NEXT));

keywordMap.put("no", new Integer(SqlParserSymbols.KW_NO));

keywordMap.put("not", new Integer(SqlParserSymbols.KW_NOT));

keywordMap.put("null", new Integer(SqlParserSymbols.KW_NULL));

keywordMap.put("nulls", new Integer(SqlParserSymbols.KW_NULLS));

keywordMap.put("observer", new Integer(SqlParserSymbols.KW_OBSERVER));

keywordMap.put("of", new Integer(SqlParserSymbols.KW_OF));

keywordMap.put("offset", new Integer(SqlParserSymbols.KW_OFFSET));

keywordMap.put("on", new Integer(SqlParserSymbols.KW_ON));

keywordMap.put("only", new Integer(SqlParserSymbols.KW_ONLY));

keywordMap.put("open", new Integer(SqlParserSymbols.KW_OPEN));

keywordMap.put("or", new Integer(SqlParserSymbols.KW_OR));

keywordMap.put("order", new Integer(SqlParserSymbols.KW_ORDER));

keywordMap.put("outer", new Integer(SqlParserSymbols.KW_OUTER));

keywordMap.put("outfile", new Integer(SqlParserSymbols.KW_OUTFILE));

keywordMap.put("over", new Integer(SqlParserSymbols.KW_OVER));

keywordMap.put("overwrite", new Integer(SqlParserSymbols.KW_OVERWRITE));

keywordMap.put("parameter", new Integer(SqlParserSymbols.KW_PARAMETER));

keywordMap.put("partition", new Integer(SqlParserSymbols.KW_PARTITION));

keywordMap.put("partitions", new Integer(SqlParserSymbols.KW_PARTITIONS));

keywordMap.put("password", new Integer(SqlParserSymbols.KW_PASSWORD));

keywordMap.put("password_expire", new Integer(SqlParserSymbols.KW_PASSWORD_EXPIRE));

keywordMap.put("password_history", new Integer(SqlParserSymbols.KW_PASSWORD_HISTORY));

keywordMap.put("password_reuse", new Integer(SqlParserSymbols.KW_PASSWORD_REUSE));

keywordMap.put("password_lock_time", new Integer(SqlParserSymbols.KW_PASSWORD_LOCK_TIME));

keywordMap.put("path", new Integer(SqlParserSymbols.KW_PATH));

keywordMap.put("pause", new Integer(SqlParserSymbols.KW_PAUSE));

keywordMap.put("period", new Integer(SqlParserSymbols.KW_PERIOD));

keywordMap.put("plugin", new Integer(SqlParserSymbols.KW_PLUGIN));

keywordMap.put("plugins", new Integer(SqlParserSymbols.KW_PLUGINS));

keywordMap.put("policy", new Integer(SqlParserSymbols.KW_POLICY));

keywordMap.put("preceding", new Integer(SqlParserSymbols.KW_PRECEDING));

keywordMap.put("percent", new Integer(SqlParserSymbols.KW_PERCENT));

keywordMap.put("privileges", new Integer(SqlParserSymbols.KW_PRIVILEGES));

keywordMap.put("proc", new Integer(SqlParserSymbols.KW_PROC));

keywordMap.put("procedure", new Integer(SqlParserSymbols.KW_PROCEDURE));

keywordMap.put("processlist", new Integer(SqlParserSymbols.KW_PROCESSLIST));

keywordMap.put("profile", new Integer(SqlParserSymbols.KW_PROFILE));

keywordMap.put("properties", new Integer(SqlParserSymbols.KW_PROPERTIES));

keywordMap.put("conditions", new Integer(SqlParserSymbols.KW_CONDITIONS));

keywordMap.put("actions", new Integer(SqlParserSymbols.KW_ACTIONS));

keywordMap.put("set_session_variable", new Integer(SqlParserSymbols.KW_SET_SESSION_VAR));

keywordMap.put("property", new Integer(SqlParserSymbols.KW_PROPERTY));

keywordMap.put("quantile_state", new Integer(SqlParserSymbols.KW_QUANTILE_STATE));

keywordMap.put("quantile_union", new Integer(SqlParserSymbols.KW_QUANTILE_UNION));

keywordMap.put("agg_state", new Integer(SqlParserSymbols.KW_AGG_STATE));

keywordMap.put("query", new Integer(SqlParserSymbols.KW_QUERY));

keywordMap.put("quota", new Integer(SqlParserSymbols.KW_QUOTA));

keywordMap.put("random", new Integer(SqlParserSymbols.KW_RANDOM));

keywordMap.put("range", new Integer(SqlParserSymbols.KW_RANGE));

keywordMap.put("read", new Integer(SqlParserSymbols.KW_READ));

keywordMap.put("real", new Integer(SqlParserSymbols.KW_DOUBLE));

keywordMap.put("rebalance", new Integer(SqlParserSymbols.KW_REBALANCE));

keywordMap.put("recent", new Integer(SqlParserSymbols.KW_RECENT));

keywordMap.put("recover", new Integer(SqlParserSymbols.KW_RECOVER));

keywordMap.put("recycle", new Integer(SqlParserSymbols.KW_RECYCLE));

keywordMap.put("refresh", new Integer(SqlParserSymbols.KW_REFRESH));

keywordMap.put("regexp", new Integer(SqlParserSymbols.KW_REGEXP));

keywordMap.put("release", new Integer(SqlParserSymbols.KW_RELEASE));

keywordMap.put("rename", new Integer(SqlParserSymbols.KW_RENAME));

keywordMap.put("repair", new Integer(SqlParserSymbols.KW_REPAIR));

keywordMap.put("repeatable", new Integer(SqlParserSymbols.KW_REPEATABLE));

keywordMap.put("replace", new Integer(SqlParserSymbols.KW_REPLACE));

keywordMap.put("replace_if_not_null", new Integer(SqlParserSymbols.KW_REPLACE_IF_NOT_NULL));

keywordMap.put("replica", new Integer(SqlParserSymbols.KW_REPLICA));

keywordMap.put("repositories", new Integer(SqlParserSymbols.KW_REPOSITORIES));

keywordMap.put("repository", new Integer(SqlParserSymbols.KW_REPOSITORY));

keywordMap.put("resource", new Integer(SqlParserSymbols.KW_RESOURCE));

keywordMap.put("resources", new Integer(SqlParserSymbols.KW_RESOURCES));

keywordMap.put("restore", new Integer(SqlParserSymbols.KW_RESTORE));

keywordMap.put("resume", new Integer(SqlParserSymbols.KW_RESUME));

keywordMap.put("returns", new Integer(SqlParserSymbols.KW_RETURNS));

keywordMap.put("revoke", new Integer(SqlParserSymbols.KW_REVOKE));

keywordMap.put("right", new Integer(SqlParserSymbols.KW_RIGHT));

keywordMap.put("rlike", new Integer(SqlParserSymbols.KW_REGEXP));

keywordMap.put("role", new Integer(SqlParserSymbols.KW_ROLE));

keywordMap.put("roles", new Integer(SqlParserSymbols.KW_ROLES));

keywordMap.put("rollback", new Integer(SqlParserSymbols.KW_ROLLBACK));

keywordMap.put("rollup", new Integer(SqlParserSymbols.KW_ROLLUP));

keywordMap.put("routine", new Integer(SqlParserSymbols.KW_ROUTINE));

keywordMap.put("row", new Integer(SqlParserSymbols.KW_ROW));

keywordMap.put("rows", new Integer(SqlParserSymbols.KW_ROWS));

keywordMap.put("s3", new Integer(SqlParserSymbols.KW_S3));

keywordMap.put("schema", new Integer(SqlParserSymbols.KW_SCHEMA));

keywordMap.put("schedule", new Integer(SqlParserSymbols.KW_SCHEDULE));

keywordMap.put("schemas", new Integer(SqlParserSymbols.KW_SCHEMAS));

keywordMap.put("second", new Integer(SqlParserSymbols.KW_SECOND));

keywordMap.put("select", new Integer(SqlParserSymbols.KW_SELECT));

keywordMap.put("semi", new Integer(SqlParserSymbols.KW_SEMI));

keywordMap.put("serializable", new Integer(SqlParserSymbols.KW_SERIALIZABLE));

keywordMap.put("session", new Integer(SqlParserSymbols.KW_SESSION));

keywordMap.put("set", new Integer(SqlParserSymbols.KW_SET));

keywordMap.put("sets", new Integer(SqlParserSymbols.KW_SETS));

keywordMap.put("show", new Integer(SqlParserSymbols.KW_SHOW));

keywordMap.put("signed", new Integer(SqlParserSymbols.KW_SIGNED));

keywordMap.put("skew", new Integer(SqlParserSymbols.KW_SKEW));

keywordMap.put("smallint", new Integer(SqlParserSymbols.KW_SMALLINT));

keywordMap.put("snapshot", new Integer(SqlParserSymbols.KW_SNAPSHOT));

keywordMap.put("soname", new Integer(SqlParserSymbols.KW_SONAME));

keywordMap.put("split", new Integer(SqlParserSymbols.KW_SPLIT));

keywordMap.put("sql", new Integer(SqlParserSymbols.KW_SQL));

keywordMap.put("sql_block_rule", new Integer(SqlParserSymbols.KW_SQL_BLOCK_RULE));

keywordMap.put("sample", new Integer(SqlParserSymbols.KW_SAMPLE));

keywordMap.put("start", new Integer(SqlParserSymbols.KW_START));

keywordMap.put("starts", new Integer(SqlParserSymbols.KW_STARTS));

keywordMap.put("stats", new Integer(SqlParserSymbols.KW_STATS));

keywordMap.put("status", new Integer(SqlParserSymbols.KW_STATUS));

keywordMap.put("stop", new Integer(SqlParserSymbols.KW_STOP));

keywordMap.put("storage", new Integer(SqlParserSymbols.KW_STORAGE));

keywordMap.put("vault", new Integer(SqlParserSymbols.KW_VAULT));

keywordMap.put("vaults", new Integer(SqlParserSymbols.KW_VAULTS));

keywordMap.put("stream", new Integer(SqlParserSymbols.KW_STREAM));

keywordMap.put("streaming", new Integer(SqlParserSymbols.KW_STREAMING));

keywordMap.put("string", new Integer(SqlParserSymbols.KW_STRING));

keywordMap.put("struct", new Integer(SqlParserSymbols.KW_STRUCT));

keywordMap.put("sum", new Integer(SqlParserSymbols.KW_SUM));

keywordMap.put("superuser", new Integer(SqlParserSymbols.KW_SUPERUSER));

keywordMap.put("switch", new Integer(SqlParserSymbols.KW_SWITCH));

keywordMap.put("sync", new Integer(SqlParserSymbols.KW_SYNC));

keywordMap.put("system", new Integer(SqlParserSymbols.KW_SYSTEM));

keywordMap.put("table", new Integer(SqlParserSymbols.KW_TABLE));

keywordMap.put("tables", new Integer(SqlParserSymbols.KW_TABLES));

keywordMap.put("tablesample", new Integer(SqlParserSymbols.KW_TABLESAMPLE));

keywordMap.put("tablet", new Integer(SqlParserSymbols.KW_TABLET));

keywordMap.put("tablets", new Integer(SqlParserSymbols.KW_TABLETS));

keywordMap.put("task", new Integer(SqlParserSymbols.KW_TASK));

keywordMap.put("tasks", new Integer(SqlParserSymbols.KW_TASKS));

keywordMap.put("temporary", new Integer(SqlParserSymbols.KW_TEMPORARY));

keywordMap.put("terminated", new Integer(SqlParserSymbols.KW_TERMINATED));

keywordMap.put("text", new Integer(SqlParserSymbols.KW_TEXT));

keywordMap.put("than", new Integer(SqlParserSymbols.KW_THAN));

keywordMap.put("then", new Integer(SqlParserSymbols.KW_THEN));

keywordMap.put("timestamp", new Integer(SqlParserSymbols.KW_TIMESTAMP));

keywordMap.put("tinyint", new Integer(SqlParserSymbols.KW_TINYINT));

keywordMap.put("to", new Integer(SqlParserSymbols.KW_TO));

keywordMap.put("transaction", new Integer(SqlParserSymbols.KW_TRANSACTION));

keywordMap.put("trash", new Integer(SqlParserSymbols.KW_TRASH));

keywordMap.put("triggers", new Integer(SqlParserSymbols.KW_TRIGGERS));

keywordMap.put("trim", new Integer(SqlParserSymbols.KW_TRIM));

keywordMap.put("true", new Integer(SqlParserSymbols.KW_TRUE));

keywordMap.put("truncate", new Integer(SqlParserSymbols.KW_TRUNCATE));

keywordMap.put("type", new Integer(SqlParserSymbols.KW_TYPE));

keywordMap.put("types", new Integer(SqlParserSymbols.KW_TYPES));

keywordMap.put("unbounded", new Integer(SqlParserSymbols.KW_UNBOUNDED));

keywordMap.put("uncommitted", new Integer(SqlParserSymbols.KW_UNCOMMITTED));

keywordMap.put("uninstall", new Integer(SqlParserSymbols.KW_UNINSTALL));

keywordMap.put("union", new Integer(SqlParserSymbols.KW_UNION));

keywordMap.put("unique", new Integer(SqlParserSymbols.KW_UNIQUE));

keywordMap.put("unlock", new Integer(SqlParserSymbols.KW_UNLOCK));

keywordMap.put("unsigned", new Integer(SqlParserSymbols.KW_UNSIGNED));

keywordMap.put("update", new Integer(SqlParserSymbols.KW_UPDATE));

keywordMap.put("use", new Integer(SqlParserSymbols.KW_USE));

keywordMap.put("user", new Integer(SqlParserSymbols.KW_USER));

keywordMap.put("using", new Integer(SqlParserSymbols.KW_USING));

keywordMap.put("value", new Integer(SqlParserSymbols.KW_VALUE));

keywordMap.put("values", new Integer(SqlParserSymbols.KW_VALUES));

keywordMap.put("varchar", new Integer(SqlParserSymbols.KW_VARCHAR));

keywordMap.put("variables", new Integer(SqlParserSymbols.KW_VARIABLES));

keywordMap.put("verbose", new Integer(SqlParserSymbols.KW_VERBOSE));

keywordMap.put("version", new Integer(SqlParserSymbols.KW_VERSION));

keywordMap.put("view", new Integer(SqlParserSymbols.KW_VIEW));

keywordMap.put("views", new Integer(SqlParserSymbols.KW_VIEWS));

keywordMap.put("warnings", new Integer(SqlParserSymbols.KW_WARNINGS));

keywordMap.put("week", new Integer(SqlParserSymbols.KW_WEEK));

keywordMap.put("when", new Integer(SqlParserSymbols.KW_WHEN));

keywordMap.put("where", new Integer(SqlParserSymbols.KW_WHERE));

keywordMap.put("whitelist", new Integer(SqlParserSymbols.KW_WHITELIST));

keywordMap.put("with", new Integer(SqlParserSymbols.KW_WITH));

keywordMap.put("work", new Integer(SqlParserSymbols.KW_WORK));

keywordMap.put("workload", new Integer(SqlParserSymbols.KW_WORKLOAD));

keywordMap.put("write", new Integer(SqlParserSymbols.KW_WRITE));

keywordMap.put("year", new Integer(SqlParserSymbols.KW_YEAR));

keywordMap.put("stage", new Integer(SqlParserSymbols.KW_STAGE));

keywordMap.put("stages", new Integer(SqlParserSymbols.KW_STAGES));

keywordMap.put("mtmv", new Integer(SqlParserSymbols.KW_MTMV));

keywordMap.put("histogram", new Integer(SqlParserSymbols.KW_HISTOGRAM));

keywordMap.put("auto", new Integer(SqlParserSymbols.KW_AUTO));

keywordMap.put("prepare", new Integer(SqlParserSymbols.KW_PREPARE));

keywordMap.put("execute", new Integer(SqlParserSymbols.KW_EXECUTE));

keywordMap.put("warm", new Integer(SqlParserSymbols.KW_WARM));

keywordMap.put("up", new Integer(SqlParserSymbols.KW_UP));

keywordMap.put("lines", new Integer(SqlParserSymbols.KW_LINES));

keywordMap.put("ignore", new Integer(SqlParserSymbols.KW_IGNORE));

keywordMap.put("expired", new Integer(SqlParserSymbols.KW_EXPIRED));

keywordMap.put("cron", new Integer(SqlParserSymbols.KW_CRON));

keywordMap.put("convert_light_schema_change_process", new Integer(SqlParserSymbols.KW_CONVERT_LSC));

keywordMap.put("unset", new Integer(SqlParserSymbols.KW_UNSET));

keywordMap.put("variable", new Integer(SqlParserSymbols.KW_VARIABLE));

}

// map from token id to token description

public static final Map<Integer, String> tokenIdMap =

new HashMap<Integer, String>();

static {

Iterator<Map.Entry<String, Integer>> it = keywordMap.entrySet().iterator();

while (it.hasNext()) {

Map.Entry<String, Integer> pairs = (Map.Entry<String, Integer>) it.next();

tokenIdMap.put(pairs.getValue(), pairs.getKey().toUpperCase());

}

// add non-keyword tokens

tokenIdMap.put(new Integer(SqlParserSymbols.IDENT), "IDENTIFIER");

tokenIdMap.put(new Integer(SqlParserSymbols.COMMA), "COMMA");

tokenIdMap.put(new Integer(SqlParserSymbols.ARROW), "->");

tokenIdMap.put(new Integer(SqlParserSymbols.BITNOT), "~");

tokenIdMap.put(new Integer(SqlParserSymbols.LPAREN), "(");

tokenIdMap.put(new Integer(SqlParserSymbols.RPAREN), ")");

tokenIdMap.put(new Integer(SqlParserSymbols.LBRACKET), "[");

tokenIdMap.put(new Integer(SqlParserSymbols.RBRACKET), "]");

tokenIdMap.put(new Integer(SqlParserSymbols.LBRACE), "{");

tokenIdMap.put(new Integer(SqlParserSymbols.RBRACE), "}");

tokenIdMap.put(new Integer(SqlParserSymbols.COLON), ":");

tokenIdMap.put(new Integer(SqlParserSymbols.SEMICOLON), ";");

tokenIdMap.put(new Integer(SqlParserSymbols.FLOATINGPOINT_LITERAL),

"FLOATING POINT LITERAL");

tokenIdMap.put(new Integer(SqlParserSymbols.INTEGER_LITERAL), "INTEGER LITERAL");

tokenIdMap.put(new Integer(SqlParserSymbols.LARGE_INTEGER_LITERAL), "INTEGER LITERAL");

tokenIdMap.put(new Integer(SqlParserSymbols.DECIMAL_LITERAL), "DECIMAL LITERAL");

tokenIdMap.put(new Integer(SqlParserSymbols.NOT), "!");

tokenIdMap.put(new Integer(SqlParserSymbols.LESSTHAN), "<");

tokenIdMap.put(new Integer(SqlParserSymbols.GREATERTHAN), ">");

tokenIdMap.put(new Integer(SqlParserSymbols.UNMATCHED_STRING_LITERAL),

"UNMATCHED STRING LITERAL");

tokenIdMap.put(new Integer(SqlParserSymbols.MOD), "%");

tokenIdMap.put(new Integer(SqlParserSymbols.ADD), "+");

tokenIdMap.put(new Integer(SqlParserSymbols.DIVIDE), "/");

tokenIdMap.put(new Integer(SqlParserSymbols.EQUAL), "=");

tokenIdMap.put(new Integer(SqlParserSymbols.STAR), "*");

tokenIdMap.put(new Integer(SqlParserSymbols.AT), "@");

tokenIdMap.put(new Integer(SqlParserSymbols.BITOR), "|");

tokenIdMap.put(new Integer(SqlParserSymbols.DOTDOTDOT), "...");

tokenIdMap.put(new Integer(SqlParserSymbols.DOT), ".");

tokenIdMap.put(new Integer(SqlParserSymbols.STRING_LITERAL), "STRING LITERAL");

tokenIdMap.put(new Integer(SqlParserSymbols.EOF), "EOF");

tokenIdMap.put(new Integer(SqlParserSymbols.SUBTRACT), "-");

tokenIdMap.put(new Integer(SqlParserSymbols.BITAND), "&");

tokenIdMap.put(new Integer(SqlParserSymbols.error), "ERROR");

tokenIdMap.put(new Integer(SqlParserSymbols.BITXOR), "^");

tokenIdMap.put(new Integer(SqlParserSymbols.NUMERIC_OVERFLOW), "NUMERIC OVERFLOW");

tokenIdMap.put(new Integer(SqlParserSymbols.PLACEHOLDER), "?");

}

public static boolean isKeyword(Integer tokenId) {

String token = tokenIdMap.get(tokenId);

return keywordMap.containsKey(token);

/* return keywordMap.containsKey(token.toLowerCase()); */

}

public static boolean isKeyword(String str) {

if (str == null) {

return false;

}

return keywordMap.containsKey(str.toLowerCase());

}

private Symbol newToken(int id, Object value) {

return new Symbol(id, yyline+1, yycolumn+1, value);

}

private static String escapeBackSlash(String str) {

StringWriter writer = new StringWriter();

int strLen = str.length();

for (int i = 0; i < strLen; ++i) {

char c = str.charAt(i);

if (c == '\\' && (i + 1) < strLen) {

switch (str.charAt(i + 1)) {

case 'n':

writer.append('\n');

break;

case 't':

writer.append('\t');

break;

case 'r':

writer.append('\r');

break;

case 'b':

writer.append('\b');

break;

case '0':

writer.append('\0'); // Ascii null

break;

case 'Z': // ^Z must be escaped on Win32

writer.append('\032');

break;

case '_':

case '%':

writer.append('\\'); // remember prefix for wildcard

/* Fall through */

default:

writer.append(str.charAt(i + 1));

break;

}

i++;

} else {

writer.append(c);

}

}

return writer.toString();

}

%}

%init{

this.sql_mode = sql_mode;

%init}

LineTerminator = \r|\n|\r\n

Whitespace = {LineTerminator} | [ \t\f]

IdentifierOrKwContents = [:digit:]*[:jletter:][:jletterdigit:]* | "&&" | "||"

IdentifierOrKw = \`{IdentifierOrKwContents}\` | {IdentifierOrKwContents}

IntegerLiteral = [:digit:][:digit:]*

QuotedIdentifier = \`(\`\`|[^\`])*\`

SingleQuoteStringLiteral = \'(\\.|[^\\\']|\'\')*\'

DoubleQuoteStringLiteral = \"(\\.|[^\\\"]|\"\")*\"

FLit1 = [0-9]+ \. [0-9]*

FLit2 = \. [0-9]+

FLit3 = [0-9]+

Exponent = [eE] [+-]? [0-9]+

DoubleLiteral = ({FLit1}|{FLit2}|{FLit3})

ExponentLiteral = ({FLit1}|{FLit2}|{FLit3}) {Exponent}

EolHintBegin = "--" " "* "+"

CommentedHintBegin = "/*" " "* "+"

CommentedHintEnd = "*/"

// Both types of plan hints must appear within a single line.

HintContent = " "* "+" [^\r\n]*

Comment = {TraditionalComment} | {EndOfLineComment}

// Match anything that has a comment end (*/) in it.

ContainsCommentEnd = [^]* "*/" [^]*

// Match anything that has a line terminator in it.

ContainsLineTerminator = [^]* {LineTerminator} [^]*

// A traditional comment is anything that starts and ends like a comment and has neither a

// plan hint inside nor a CommentEnd (*/).

TraditionalComment = "/*" !({HintContent}|{ContainsCommentEnd}) "*/"

// Similar for a end-of-line comment.

EndOfLineComment = "--" !({HintContent}|{ContainsLineTerminator}) {LineTerminator}?

// This additional state is needed because newlines signal the end of a end-of-line hint

// if one has been started earlier. Hence we need to discern between newlines within and

// outside of end-of-line hints.

%state EOLHINT

%%

"..." { return newToken(SqlParserSymbols.DOTDOTDOT, null); }

"->" { return newToken(SqlParserSymbols.ARROW, null); }

// single-character tokens

"," { return newToken(SqlParserSymbols.COMMA, null); }

"." { return newToken(SqlParserSymbols.DOT, null); }

"*" { return newToken(SqlParserSymbols.STAR, null); }

"@" { return newToken(SqlParserSymbols.AT, null); }

"(" { return newToken(SqlParserSymbols.LPAREN, null); }

")" { return newToken(SqlParserSymbols.RPAREN, null); }

":" { return newToken(SqlParserSymbols.COLON, null); }

";" { return newToken(SqlParserSymbols.SEMICOLON, null); }

"[" { return newToken(SqlParserSymbols.LBRACKET, null); }

"]" { return newToken(SqlParserSymbols.RBRACKET, null); }

"{" { return newToken(SqlParserSymbols.LBRACE, null); }

"}" { return newToken(SqlParserSymbols.RBRACE, null); }

"/" { return newToken(SqlParserSymbols.DIVIDE, null); }

"%" { return newToken(SqlParserSymbols.MOD, null); }

"+" { return newToken(SqlParserSymbols.ADD, null); }

"-" { return newToken(SqlParserSymbols.SUBTRACT, null); }

"&" { return newToken(SqlParserSymbols.BITAND, null); }

"|" { return newToken(SqlParserSymbols.BITOR, null); }

"^" { return newToken(SqlParserSymbols.BITXOR, null); }

"~" { return newToken(SqlParserSymbols.BITNOT, null); }

"=" { return newToken(SqlParserSymbols.EQUAL, null); }

":=" { return newToken(SqlParserSymbols.SET_VAR, null); }

"!" { return newToken(SqlParserSymbols.NOT, null); }

"<" { return newToken(SqlParserSymbols.LESSTHAN, null); }

">" { return newToken(SqlParserSymbols.GREATERTHAN, null); }

"\"" { return newToken(SqlParserSymbols.UNMATCHED_STRING_LITERAL, null); }

"'" { return newToken(SqlParserSymbols.UNMATCHED_STRING_LITERAL, null); }

"`" { return newToken(SqlParserSymbols.UNMATCHED_STRING_LITERAL, null); }

"?" { return newToken(SqlParserSymbols.PLACEHOLDER, null); }

{ExponentLiteral} {

BigDecimal decimal_val;

try {

decimal_val = new BigDecimal(yytext());

} catch (NumberFormatException e) {

return newToken(SqlParserSymbols.NUMERIC_OVERFLOW, yytext());

}

return newToken(SqlParserSymbols.DECIMAL_LITERAL, decimal_val);

}

{QuotedIdentifier} {

// Remove the quotes

String trimmedIdent = yytext().substring(1, yytext().length() - 1);

return newToken(SqlParserSymbols.IDENT, SqlUtils.escapeUnquote(trimmedIdent));

}

{IdentifierOrKw} {

String text = yytext();

Integer kw_id = keywordMap.get(text.toLowerCase());

/* Integer kw_id = keywordMap.get(text); */

if (kw_id != null) {

// if MODE_PIPES_AS_CONCAT is not active, treat '||' symbol as same as 'or' symbol

if ((kw_id == SqlParserSymbols.KW_PIPE) &&

((this.sql_mode & SqlModeHelper.MODE_PIPES_AS_CONCAT) == 0)) {

return newToken(SqlParserSymbols.KW_OR, text);

}

return newToken(kw_id.intValue(), text);

} else {

return newToken(SqlParserSymbols.IDENT, text);

}

}

{SingleQuoteStringLiteral} {

return newToken(SqlParserSymbols.STRING_LITERAL,

escapeBackSlash(yytext().substring(1, yytext().length()-1)).replaceAll("''", "'"));

}

{DoubleQuoteStringLiteral} {

return newToken(SqlParserSymbols.STRING_LITERAL,

escapeBackSlash(yytext().substring(1, yytext().length()-1)).replaceAll("\"\"", "\""));

}

{CommentedHintBegin} {

return newToken(SqlParserSymbols.COMMENTED_PLAN_HINT_START, null);

}

{CommentedHintEnd} {

return newToken(SqlParserSymbols.COMMENTED_PLAN_HINT_END, null);

}

{EolHintBegin} {

yybegin(EOLHINT);

return newToken(SqlParserSymbols.COMMENTED_PLAN_HINT_START, null);

}

<EOLHINT> {LineTerminator} {

yybegin(YYINITIAL);

return newToken(SqlParserSymbols.COMMENTED_PLAN_HINT_END, null);

}

{IntegerLiteral} {

BigInteger val = null;

try {

val = new BigInteger(yytext());

} catch (NumberFormatException e) {

return newToken(SqlParserSymbols.NUMERIC_OVERFLOW, yytext());

}

// Note: val is positive, because we do not recognize minus charactor in 'IntegerLiteral'

// -2^63 will be recognize as largeint(__int128)

if (val.compareTo(LONG_MAX) <= 0) {

return newToken(SqlParserSymbols.INTEGER_LITERAL, val.longValue());

}

if (val.compareTo(LARGEINT_MAX_ABS) <= 0) {

return newToken(SqlParserSymbols.LARGE_INTEGER_LITERAL, val.toString());

}

return newToken(SqlParserSymbols.NUMERIC_OVERFLOW, yytext());

}

{DoubleLiteral} {

BigDecimal decimal_val;

try {

decimal_val = new BigDecimal(yytext());

} catch (NumberFormatException e) {

return newToken(SqlParserSymbols.NUMERIC_OVERFLOW, yytext());

}

return newToken(SqlParserSymbols.DECIMAL_LITERAL, decimal_val);

}

{Comment} { /* ignore */ }

{Whitespace} { /* ignore */ }