Skip to content

Commit 84d23e7

Browse files
fudianchnfudianchn
authored
feat(parser): concatenate adjacent string literals behind adjacentStringLiterals (#2514)
* feat(parser): concatenate adjacent string literals behind adjacentStringLiterals Three readings of two neighboring string literals: alias (MySQL, SQL Server, the current behavior), concatenated across a newline (the SQL standard and PostgreSQL), concatenated across any whitespace (GoogleSQL, Spark/Databricks). The mode lives in a String valued Feature like Feature.dialect, the merge is one guarded loop in PrimaryExpression using the token line numbers for the newline rule, producing a single merged StringValue. ANSI_SQL and POSTGRESQL presets carry NEWLINE. Implements item 3 of #2512. * docs(site): document adjacentStringLiterals in the parser features section * hardening: null-safe mode check in isAdjacentStringConcat The FeatureConfiguration seeds every configurable Feature with its declared default, so the mode is never null today; the guard keeps the predicate locally correct regardless. * feat(parser): withAdjacentStringLiterals(boolean) mapping false = OFF, true = NEWLINE stores the AdjacentStringLiterals enum constant itself as the feature value * docs(parser): move isAdjacentStringConcat below isNamedExprListAhead, reattaching its javadoc
1 parent c86cf6a commit 84d23e7

5 files changed

Lines changed: 155 additions & 7 deletions

File tree

‎src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java‎

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,24 +24,41 @@ public abstract class AbstractJSqlParser<P> {
2424
protected boolean errorRecovery = false;
2525
protected List<ParseException> parseErrors = new ArrayList<>();
2626

27+
public enum AdjacentStringLiterals {
28+
OFF, NEWLINE, WHITESPACE
29+
}
30+
2731
public enum Dialect {
28-
ANSI_SQL, ORACLE, MYSQL(Feature.allowBackslashEscapeCharacter,
32+
ANSI_SQL(AdjacentStringLiterals.NEWLINE), ORACLE, MYSQL(AdjacentStringLiterals.OFF,
33+
Feature.allowBackslashEscapeCharacter,
2934
Feature.allowHashLineComments,
30-
Feature.allowDoubleQuotedStrings), MARIADB(Feature.allowBackslashEscapeCharacter,
35+
Feature.allowDoubleQuotedStrings), MARIADB(AdjacentStringLiterals.OFF,
36+
Feature.allowBackslashEscapeCharacter,
3137
Feature.allowHashLineComments,
32-
Feature.allowDoubleQuotedStrings), SQLSERVER(
33-
Feature.allowSquareBracketQuotation), POSTGRESQL, H2, EXASOL;
38+
Feature.allowDoubleQuotedStrings), SQLSERVER(AdjacentStringLiterals.OFF,
39+
Feature.allowSquareBracketQuotation), POSTGRESQL(
40+
AdjacentStringLiterals.NEWLINE), H2, EXASOL;
3441

3542
private final Set<Feature> lexerFeatures;
43+
private final AdjacentStringLiterals adjacentStringLiterals;
3644

37-
Dialect(Feature... lexerFeatures) {
45+
Dialect(AdjacentStringLiterals adjacentStringLiterals, Feature... lexerFeatures) {
46+
this.adjacentStringLiterals = adjacentStringLiterals;
3847
this.lexerFeatures = lexerFeatures.length == 0 ? EnumSet.noneOf(Feature.class)
3948
: EnumSet.copyOf(Arrays.asList(lexerFeatures));
4049
}
4150

51+
Dialect(Feature... lexerFeatures) {
52+
this(AdjacentStringLiterals.OFF, lexerFeatures);
53+
}
54+
4255
public Set<Feature> getLexerFeatures() {
4356
return lexerFeatures;
4457
}
58+
59+
public AdjacentStringLiterals getAdjacentStringLiterals() {
60+
return adjacentStringLiterals;
61+
}
4562
}
4663

4764
public P withSquareBracketQuotation() {
@@ -82,12 +99,30 @@ public P withTimeOut(long timeOutMillSeconds) {
8299

83100
public P withDialect(Dialect dialect) {
84101
withFeature(Feature.dialect, dialect.name());
102+
if (dialect.getAdjacentStringLiterals() != AdjacentStringLiterals.OFF) {
103+
withAdjacentStringLiterals(dialect.getAdjacentStringLiterals());
104+
}
85105
for (Feature lexerFeature : dialect.getLexerFeatures()) {
86106
withFeature(lexerFeature, true);
87107
}
88108
return me();
89109
}
90110

111+
public P withAdjacentStringLiterals() {
112+
return withAdjacentStringLiterals(AdjacentStringLiterals.NEWLINE);
113+
}
114+
115+
public P withAdjacentStringLiterals(boolean adjacentStringLiterals) {
116+
return withAdjacentStringLiterals(
117+
adjacentStringLiterals ? AdjacentStringLiterals.NEWLINE
118+
: AdjacentStringLiterals.OFF);
119+
}
120+
121+
public P withAdjacentStringLiterals(AdjacentStringLiterals adjacentStringLiterals) {
122+
getConfiguration().setValue(Feature.adjacentStringLiterals, adjacentStringLiterals);
123+
return me();
124+
}
125+
91126
public P withAllowedNestingDepth(int allowedNestingDepth) {
92127
return withFeature(Feature.allowedNestingDepth, allowedNestingDepth);
93128
}

‎src/main/java/net/sf/jsqlparser/parser/feature/Feature.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import net.sf.jsqlparser.expression.OracleHierarchicalExpression;
1616
import net.sf.jsqlparser.expression.OracleHint;
1717
import net.sf.jsqlparser.expression.operators.relational.SupportsOldOracleJoinSyntax;
18+
import net.sf.jsqlparser.parser.AbstractJSqlParser;
1819
import net.sf.jsqlparser.statement.Block;
1920
import net.sf.jsqlparser.statement.Commit;
2021
import net.sf.jsqlparser.statement.CreateFunctionalStatement;
@@ -801,6 +802,14 @@ public enum Feature {
801802
*/
802803
allowDoubleQuotedStrings(false),
803804

805+
/**
806+
* concatenates adjacent String Literals: NEWLINE when separated by whitespace with at least one
807+
* newline (the SQL standard and PostgreSQL), WHITESPACE across any whitespace (GoogleSQL,
808+
* Spark/Databricks); OFF by default, where the second literal stays an alias (MySQL, SQL
809+
* Server) or fails (everywhere else)
810+
*/
811+
adjacentStringLiterals(AbstractJSqlParser.AdjacentStringLiterals.OFF),
812+
804813
/**
805814
* allows MySQL `#` line comments; disabled by default, where a lone `#` stays the binary
806815
* operator (#2507: PostgreSQL bitwise XOR / geometric intersection)

‎src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1127,6 +1127,24 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
11271127
}
11281128
}
11291129

1130+
/**
1131+
* Semantic lookahead for the adjacent string literal concatenation in PrimaryExpression:
1132+
* true when Feature.adjacentStringLiterals is on and the next token is a string literal
1133+
* meeting the mode's separation rule (WHITESPACE: any, NEWLINE: a newline between the
1134+
* tokens, detected through the token line numbers).
1135+
*/
1136+
protected boolean isAdjacentStringConcat() {
1137+
String mode = getAsString(Feature.adjacentStringLiterals);
1138+
if (mode == null || AdjacentStringLiterals.OFF.name().equals(mode)) {
1139+
return false;
1140+
}
1141+
if (getToken(1).kind != S_CHAR_LITERAL) {
1142+
return false;
1143+
}
1144+
return AdjacentStringLiterals.WHITESPACE.name().equals(mode)
1145+
|| getToken(1).beginLine > getToken(0).endLine;
1146+
}
1147+
11301148
/**
11311149
* Checks if the next token can start a condition suffix
11321150
* (comparison, IN, BETWEEN, LIKE, IS NULL, etc.)
@@ -8319,6 +8337,7 @@ Expression PrimaryExpression() #PrimaryExpression:
83198337
Expression timezoneRightExpr = null;
83208338
Token token = null;
83218339
Token sign = null;
8340+
Token adjacentToken = null;
83228341
String tmp = "";
83238342
ColDataType type = null;
83248343
boolean not = false;
@@ -8419,6 +8438,13 @@ Expression PrimaryExpression() #PrimaryExpression:
84198438
| LOOKAHEAD(2, {!interrupted}) (token=<K_TRUE> | token=<K_FALSE>) { retval = new BooleanValue(token.image); }
84208439

84218440
| token=<S_CHAR_LITERAL> { retval = new StringValue(token.image); linkAST(retval,jjtThis); }
8441+
( LOOKAHEAD({ isAdjacentStringConcat() })
8442+
adjacentToken=<S_CHAR_LITERAL>
8443+
{
8444+
((StringValue) retval)
8445+
.setValue(((StringValue) retval).getValue() + new StringValue(adjacentToken.image).getValue());
8446+
}
8447+
)*
84228448

84238449
| "{d" token=<S_CHAR_LITERAL> "}" { retval = new DateValue(token.image); }
84248450

‎src/site/sphinx/usage.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ Define the Parser Features
279279

280280
JSQLParser interprets Squared Brackets ``[..]`` as Arrays, which does not work with MS SQL Server and T-SQL. Please use the Parser Features to instruct JSQLParser to read Squared Brackets as Quotes instead.
281281

282-
JSQLParser allows for standard compliant Single Quote ``'..`` Escaping. Additional Back-slash ``\..`` Escaping needs to be activated by setting the ``BackSlashEscapeCharacter`` parser feature. JSQLParser reads Double Quotes ``".."`` as quoted identifiers (ANSI SQL); reading them as String Literals (BigQuery, Spark/Databricks, MySQL default sql_mode) needs the ``DoubleQuotedStrings`` parser feature.
282+
JSQLParser allows for standard compliant Single Quote ``'..`` Escaping. Additional Back-slash ``\..`` Escaping needs to be activated by setting the ``BackSlashEscapeCharacter`` parser feature. JSQLParser reads Double Quotes ``".."`` as quoted identifiers (ANSI SQL); reading them as String Literals (BigQuery, Spark/Databricks, MySQL default sql_mode) needs the ``DoubleQuotedStrings`` parser feature. Adjacent String Literals concatenate optionally: only across a newline (``NEWLINE``, the SQL standard and PostgreSQL) or across any whitespace (``WHITESPACE``, GoogleSQL and Spark/Databricks); ``withAdjacentStringLiterals(true)`` selects the standard ``NEWLINE`` mode, ``false`` switches it off.
283283

284284
Additionally there are Features to control the Parser's effort at the cost of the performance.
285285

@@ -319,7 +319,7 @@ Additionally there are Features to control the Parser's effort at the cost of th
319319
.withBackslashEscapeCharacter(true)
320320
);
321321
322-
Instead of turning the individual Parser Features on one by one, a ``Dialect`` preset selects the features of that database dialect: ``withDialect(Dialect.MYSQL)`` turns on ``withBackslashEscapeCharacter``, ``withHashLineComments`` and ``withDoubleQuotedStrings`` (MySQL and MariaDB syntax, the latter for the default sql_mode), ``withDialect(Dialect.SQLSERVER)`` turns on ``withSquareBracketQuotation``. Features set explicitly after the dialect preset win over the preset.
322+
Instead of turning the individual Parser Features on one by one, a ``Dialect`` preset selects the features of that database dialect: ``withDialect(Dialect.MYSQL)`` turns on ``withBackslashEscapeCharacter``, ``withHashLineComments`` and ``withDoubleQuotedStrings`` (MySQL and MariaDB syntax, the latter for the default sql_mode), ``withDialect(Dialect.SQLSERVER)`` turns on ``withSquareBracketQuotation``. ``withDialect(Dialect.POSTGRESQL)`` and ``withDialect(Dialect.ANSI_SQL)`` turn on the newline rule for adjacent String Literals. Features set explicitly after the dialect preset win over the preset.
323323

324324
.. code-block:: java
325325

‎src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -655,4 +655,82 @@ public void testDoubleQuotedStringsPreset() throws Exception {
655655
p -> p.withDialect(AbstractJSqlParser.Dialect.SQLSERVER))).getSelectBody();
656656
assertTrue(select.getSelectItems().get(0).getExpression() instanceof Column);
657657
}
658+
659+
@Test
660+
public void testAdjacentStringLiteralsNewline() throws Exception {
661+
// default (off): the MySQL / SQL Server alias reading, unchanged
662+
assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a' 'b'").toString());
663+
assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'").toString());
664+
assertThrows(JSQLParserException.class,
665+
() -> CCJSqlParserUtil.parse("SELECT * FROM t WHERE x = 'a'\n'b'"));
666+
// newline mode: the standard / Postgres reading, "Two string constants
667+
// that are only separated by whitespace with at least one newline are
668+
// concatenated"
669+
PlainSelect select = (PlainSelect) ((Select) CCJSqlParserUtil.parse("SELECT 'a'\n'b'",
670+
p -> p.withAdjacentStringLiterals(
671+
AbstractJSqlParser.AdjacentStringLiterals.NEWLINE)))
672+
.getSelectBody();
673+
Expression expression = select.getSelectItems().get(0).getExpression();
674+
assertTrue(expression instanceof StringValue);
675+
assertEquals("ab", ((StringValue) expression).getValue());
676+
assertEquals("SELECT 'ab'", select.toString());
677+
// same line keeps the alias reading under newline mode
678+
assertEquals("SELECT 'a' 'b'",
679+
CCJSqlParserUtil.parse("SELECT 'a' 'b'",
680+
p -> p.withAdjacentStringLiterals(
681+
AbstractJSqlParser.AdjacentStringLiterals.NEWLINE))
682+
.toString());
683+
// expression positions concatenate too
684+
assertEquals("SELECT * FROM t WHERE x = 'ab'",
685+
CCJSqlParserUtil.parse("SELECT * FROM t WHERE x = 'a'\n'b'",
686+
p -> p.withAdjacentStringLiterals(
687+
AbstractJSqlParser.AdjacentStringLiterals.NEWLINE))
688+
.toString());
689+
// three parts merge into one literal
690+
assertEquals("SELECT 'abc'",
691+
CCJSqlParserUtil.parse("SELECT 'a'\n'b'\n'c'",
692+
p -> p.withAdjacentStringLiterals(
693+
AbstractJSqlParser.AdjacentStringLiterals.NEWLINE))
694+
.toString());
695+
// the ANSI SQL and Postgres presets carry it, the MySQL preset does not
696+
for (AbstractJSqlParser.Dialect dialect : new AbstractJSqlParser.Dialect[] {
697+
AbstractJSqlParser.Dialect.ANSI_SQL, AbstractJSqlParser.Dialect.POSTGRESQL}) {
698+
assertEquals("SELECT 'ab'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'",
699+
p -> p.withDialect(dialect)).toString());
700+
}
701+
assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'",
702+
p -> p.withDialect(AbstractJSqlParser.Dialect.MYSQL)).toString());
703+
}
704+
705+
@Test
706+
public void testAdjacentStringLiteralsWhitespace() throws Exception {
707+
// whitespace mode: the GoogleSQL / Spark reading, same line
708+
// concatenates (their literal chunking)
709+
assertEquals("SELECT 'ab'",
710+
CCJSqlParserUtil.parse("SELECT 'a' 'b'",
711+
p -> p.withAdjacentStringLiterals(
712+
AbstractJSqlParser.AdjacentStringLiterals.WHITESPACE))
713+
.toString());
714+
// combined with allowDoubleQuotedStrings: the BigQuery chunking shape
715+
assertEquals("SELECT \"12\"",
716+
CCJSqlParserUtil.parse("SELECT \"1\" \"2\"",
717+
p -> p.withDoubleQuotedStrings(true).withAdjacentStringLiterals(
718+
AbstractJSqlParser.AdjacentStringLiterals.WHITESPACE))
719+
.toString());
720+
}
721+
722+
@Test
723+
public void testAdjacentStringLiteralsBoolean() throws Exception {
724+
// true: the standard (newline) mode, the same line keeps the alias reading
725+
assertEquals("SELECT 'ab'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'",
726+
p -> p.withAdjacentStringLiterals(true)).toString());
727+
assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a' 'b'",
728+
p -> p.withAdjacentStringLiterals(true)).toString());
729+
// false: off, the alias reading also across newlines
730+
assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'",
731+
p -> p.withAdjacentStringLiterals(false)).toString());
732+
// the no-arg variant enables the standard mode
733+
assertEquals("SELECT 'ab'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'",
734+
p -> p.withAdjacentStringLiterals()).toString());
735+
}
658736
}

0 commit comments

Comments
 (0)