Skip to content

Commit 072ccc0

Browse files
vasilev-alexvasilev-alex
authored
feat: dollar-quoted strings support (apache#772)
* feat: support dollar-quoted strings * remove comment * unused code * removed debugging * added tests * fmt * clippy * updated tests
1 parent 0c9ec40 commit 072ccc0

5 files changed

Lines changed: 210 additions & 33 deletions

File tree

‎src/ast/mod.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,9 @@ pub use self::query::{
3939
SelectInto, SelectItem, SetExpr, SetOperator, SetQuantifier, Table, TableAlias, TableFactor,
4040
TableWithJoins, Top, Values, WildcardAdditionalOptions, With,
4141
};
42-
pub use self::value::{escape_quoted_string, DateTimeField, TrimWhereField, Value};
42+
pub use self::value::{
43+
escape_quoted_string, DateTimeField, DollarQuotedString, TrimWhereField, Value,
44+
};
4345

4446
#[cfg(feature = "visitor")]
4547
pub use visitor::*;

‎src/ast/value.rs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ pub enum Value {
3535
Number(BigDecimal, bool),
3636
/// 'string value'
3737
SingleQuotedString(String),
38+
// $<tag_name>$string value$<tag_name>$ (postgres syntax)
39+
DollarQuotedString(DollarQuotedString),
3840
/// e'string value' (postgres extension)
3941
/// <https://www.postgresql.org/docs/8.3/sql-syntax-lexical.html#SQL-SYNTAX-STRINGS
4042
EscapedStringLiteral(String),
@@ -60,6 +62,7 @@ impl fmt::Display for Value {
6062
Value::Number(v, l) => write!(f, "{}{long}", v, long = if *l { "L" } else { "" }),
6163
Value::DoubleQuotedString(v) => write!(f, "\"{}\"", v),
6264
Value::SingleQuotedString(v) => write!(f, "'{}'", escape_single_quote_string(v)),
65+
Value::DollarQuotedString(v) => write!(f, "{}", v),
6366
Value::EscapedStringLiteral(v) => write!(f, "E'{}'", escape_escaped_string(v)),
6467
Value::NationalStringLiteral(v) => write!(f, "N'{}'", v),
6568
Value::HexStringLiteral(v) => write!(f, "X'{}'", v),
@@ -71,6 +74,27 @@ impl fmt::Display for Value {
7174
}
7275
}
7376

77+
#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
78+
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
79+
#[cfg_attr(feature = "visitor", derive(Visit))]
80+
pub struct DollarQuotedString {
81+
pub value: String,
82+
pub tag: Option<String>,
83+
}
84+
85+
impl fmt::Display for DollarQuotedString {
86+
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
87+
match &self.tag {
88+
Some(tag) => {
89+
write!(f, "${}${}${}$", tag, self.value, tag)
90+
}
91+
None => {
92+
write!(f, "$${}$$", self.value)
93+
}
94+
}
95+
}
96+
}
97+
7498
#[derive(Debug, Copy, Clone, PartialEq, Eq, Ord, PartialOrd, Hash)]
7599
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
76100
#[cfg_attr(feature = "visitor", derive(Visit))]

‎src/parser.rs‎

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -778,6 +778,7 @@ impl<'a> Parser<'a> {
778778
Token::Number(_, _)
779779
| Token::SingleQuotedString(_)
780780
| Token::DoubleQuotedString(_)
781+
| Token::DollarQuotedString(_)
781782
| Token::NationalStringLiteral(_)
782783
| Token::HexStringLiteral(_) => {
783784
self.prev_token();
@@ -4104,6 +4105,7 @@ impl<'a> Parser<'a> {
41044105
},
41054106
Token::SingleQuotedString(ref s) => Ok(Value::SingleQuotedString(s.to_string())),
41064107
Token::DoubleQuotedString(ref s) => Ok(Value::DoubleQuotedString(s.to_string())),
4108+
Token::DollarQuotedString(ref s) => Ok(Value::DollarQuotedString(s.clone())),
41074109
Token::NationalStringLiteral(ref s) => Ok(Value::NationalStringLiteral(s.to_string())),
41084110
Token::EscapedStringLiteral(ref s) => Ok(Value::EscapedStringLiteral(s.to_string())),
41094111
Token::HexStringLiteral(ref s) => Ok(Value::HexStringLiteral(s.to_string())),
@@ -4148,24 +4150,9 @@ impl<'a> Parser<'a> {
41484150
pub fn parse_function_definition(&mut self) -> Result<FunctionDefinition, ParserError> {
41494151
let peek_token = self.peek_token();
41504152
match peek_token.token {
4151-
Token::DoubleDollarQuoting if dialect_of!(self is PostgreSqlDialect) => {
4153+
Token::DollarQuotedString(value) if dialect_of!(self is PostgreSqlDialect) => {
41524154
self.next_token();
4153-
let mut func_desc = String::new();
4154-
loop {
4155-
if let Some(next_token) = self.next_token_no_skip() {
4156-
match &next_token.token {
4157-
Token::DoubleDollarQuoting => break,
4158-
Token::EOF => {
4159-
return self.expected(
4160-
"literal string",
4161-
TokenWithLocation::wrap(Token::EOF),
4162-
);
4163-
}
4164-
token => func_desc.push_str(token.to_string().as_str()),
4165-
}
4166-
}
4167-
}
4168-
Ok(FunctionDefinition::DoubleDollarDef(func_desc))
4155+
Ok(FunctionDefinition::DoubleDollarDef(value.value))
41694156
}
41704157
_ => Ok(FunctionDefinition::SingleQuotedDef(
41714158
self.parse_literal_string()?,
@@ -4712,7 +4699,7 @@ impl<'a> Parser<'a> {
47124699
}
47134700

47144701
/// Parse a query expression, i.e. a `SELECT` statement optionally
4715-
/// preceeded with some `WITH` CTE declarations and optionally followed
4702+
/// preceded with some `WITH` CTE declarations and optionally followed
47164703
/// by `ORDER BY`. Unlike some other parse_... methods, this one doesn't
47174704
/// expect the initial keyword to be already consumed
47184705
pub fn parse_query(&mut self) -> Result<Query, ParserError> {

‎src/tokenizer.rs‎

Lines changed: 97 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ use serde::{Deserialize, Serialize};
3434
#[cfg(feature = "visitor")]
3535
use sqlparser_derive::Visit;
3636

37+
use crate::ast::DollarQuotedString;
3738
use crate::dialect::SnowflakeDialect;
3839
use crate::dialect::{Dialect, MySqlDialect};
3940
use crate::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX};
@@ -55,6 +56,8 @@ pub enum Token {
5556
SingleQuotedString(String),
5657
/// Double quoted string: i.e: "string"
5758
DoubleQuotedString(String),
59+
/// Dollar quoted string: i.e: $$string$$ or $tag_name$string$tag_name$
60+
DollarQuotedString(DollarQuotedString),
5861
/// "National" string literal: i.e: N'string'
5962
NationalStringLiteral(String),
6063
/// "escaped" string literal, which are an extension to the SQL standard: i.e: e'first \n second' or E 'first \n second'
@@ -149,8 +152,6 @@ pub enum Token {
149152
PGCubeRoot,
150153
/// `?` or `$` , a prepared statement arg placeholder
151154
Placeholder(String),
152-
/// `$$`, used for PostgreSQL create function definition
153-
DoubleDollarQuoting,
154155
/// ->, used as a operator to extract json field in PostgreSQL
155156
Arrow,
156157
/// ->>, used as a operator to extract json field as text in PostgreSQL
@@ -184,6 +185,7 @@ impl fmt::Display for Token {
184185
Token::Char(ref c) => write!(f, "{}", c),
185186
Token::SingleQuotedString(ref s) => write!(f, "'{}'", s),
186187
Token::DoubleQuotedString(ref s) => write!(f, "\"{}\"", s),
188+
Token::DollarQuotedString(ref s) => write!(f, "{}", s),
187189
Token::NationalStringLiteral(ref s) => write!(f, "N'{}'", s),
188190
Token::EscapedStringLiteral(ref s) => write!(f, "E'{}'", s),
189191
Token::HexStringLiteral(ref s) => write!(f, "X'{}'", s),
@@ -236,7 +238,6 @@ impl fmt::Display for Token {
236238
Token::HashArrow => write!(f, "#>"),
237239
Token::HashLongArrow => write!(f, "#>>"),
238240
Token::AtArrow => write!(f, "@>"),
239-
Token::DoubleDollarQuoting => write!(f, "$$"),
240241
Token::ArrowAt => write!(f, "<@"),
241242
Token::HashMinus => write!(f, "#-"),
242243
Token::AtQuestion => write!(f, "@?"),
@@ -837,17 +838,8 @@ impl<'a> Tokenizer<'a> {
837838
let s = peeking_take_while(chars, |ch| ch.is_numeric());
838839
Ok(Some(Token::Placeholder(String::from("?") + &s)))
839840
}
840-
'$' => {
841-
chars.next();
842-
match chars.peek() {
843-
Some('$') => self.consume_and_return(chars, Token::DoubleDollarQuoting),
844-
_ => {
845-
let s =
846-
peeking_take_while(chars, |ch| ch.is_alphanumeric() || ch == '_');
847-
Ok(Some(Token::Placeholder(String::from("$") + &s)))
848-
}
849-
}
850-
}
841+
'$' => Ok(Some(self.tokenize_dollar_preceded_value(chars)?)),
842+
851843
//whitespace check (including unicode chars) should be last as it covers some of the chars above
852844
ch if ch.is_whitespace() => {
853845
self.consume_and_return(chars, Token::Whitespace(Whitespace::Space))
@@ -858,6 +850,97 @@ impl<'a> Tokenizer<'a> {
858850
}
859851
}
860852

853+
/// Tokenize dollar preceded value (i.e: a string/placeholder)
854+
fn tokenize_dollar_preceded_value(&self, chars: &mut State) -> Result<Token, TokenizerError> {
855+
let mut s = String::new();
856+
let mut value = String::new();
857+
858+
chars.next();
859+
860+
if let Some('$') = chars.peek() {
861+
chars.next();
862+
863+
let mut is_terminated = false;
864+
let mut prev: Option<char> = None;
865+
866+
while let Some(&ch) = chars.peek() {
867+
if prev == Some('$') {
868+
if ch == '$' {
869+
chars.next();
870+
is_terminated = true;
871+
break;
872+
} else {
873+
s.push('$');
874+
s.push(ch);
875+
}
876+
} else if ch != '$' {
877+
s.push(ch);
878+
}
879+
880+
prev = Some(ch);
881+
chars.next();
882+
}
883+
884+
return if chars.peek().is_none() && !is_terminated {
885+
self.tokenizer_error(chars.location(), "Unterminated dollar-quoted string")
886+
} else {
887+
Ok(Token::DollarQuotedString(DollarQuotedString {
888+
value: s,
889+
tag: None,
890+
}))
891+
};
892+
} else {
893+
value.push_str(&peeking_take_while(chars, |ch| {
894+
ch.is_alphanumeric() || ch == '_'
895+
}));
896+
897+
if let Some('$') = chars.peek() {
898+
chars.next();
899+
s.push_str(&peeking_take_while(chars, |ch| ch != '$'));
900+
901+
match chars.peek() {
902+
Some('$') => {
903+
chars.next();
904+
for (_, c) in value.chars().enumerate() {
905+
let next_char = chars.next();
906+
if Some(c) != next_char {
907+
return self.tokenizer_error(
908+
chars.location(),
909+
format!(
910+
"Unterminated dollar-quoted string at or near \"{}\"",
911+
value
912+
),
913+
);
914+
}
915+
}
916+
917+
if let Some('$') = chars.peek() {
918+
chars.next();
919+
} else {
920+
return self.tokenizer_error(
921+
chars.location(),
922+
"Unterminated dollar-quoted string, expected $",
923+
);
924+
}
925+
}
926+
_ => {
927+
return self.tokenizer_error(
928+
chars.location(),
929+
"Unterminated dollar-quoted, expected $",
930+
);
931+
}
932+
}
933+
} else {
934+
return Ok(Token::Placeholder(String::from("$") + &value));
935+
}
936+
}
937+
938+
Ok(Token::DollarQuotedString(DollarQuotedString {
939+
value: s,
940+
tag: if value.is_empty() { None } else { Some(value) },
941+
}))
942+
}
943+
861944
fn tokenizer_error<R>(
862945
&self,
863946
loc: Location,

‎tests/sqlparser_postgres.rs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2507,3 +2507,84 @@ fn parse_drop_function() {
25072507
}
25082508
);
25092509
}
2510+
2511+
#[test]
2512+
fn parse_dollar_quoted_string() {
2513+
let sql = "SELECT $$hello$$, $tag_name$world$tag_name$, $$Foo$Bar$$, $$Foo$Bar$$col_name, $$$$, $tag_name$$tag_name$";
2514+
2515+
let stmt = pg().parse_sql_statements(sql).unwrap();
2516+
2517+
let projection = match stmt.get(0).unwrap() {
2518+
Statement::Query(query) => match &*query.body {
2519+
SetExpr::Select(select) => &select.projection,
2520+
_ => unreachable!(),
2521+
},
2522+
_ => unreachable!(),
2523+
};
2524+
2525+
assert_eq!(
2526+
&Expr::Value(Value::DollarQuotedString(DollarQuotedString {
2527+
tag: None,
2528+
value: "hello".into()
2529+
})),
2530+
expr_from_projection(&projection[0])
2531+
);
2532+
2533+
assert_eq!(
2534+
&Expr::Value(Value::DollarQuotedString(DollarQuotedString {
2535+
tag: Some("tag_name".into()),
2536+
value: "world".into()
2537+
})),
2538+
expr_from_projection(&projection[1])
2539+
);
2540+
2541+
assert_eq!(
2542+
&Expr::Value(Value::DollarQuotedString(DollarQuotedString {
2543+
tag: None,
2544+
value: "Foo$Bar".into()
2545+
})),
2546+
expr_from_projection(&projection[2])
2547+
);
2548+
2549+
assert_eq!(
2550+
projection[3],
2551+
SelectItem::ExprWithAlias {
2552+
expr: Expr::Value(Value::DollarQuotedString(DollarQuotedString {
2553+
tag: None,
2554+
value: "Foo$Bar".into(),
2555+
})),
2556+
alias: Ident {
2557+
value: "col_name".into(),
2558+
quote_style: None,
2559+
},
2560+
}
2561+
);
2562+
2563+
assert_eq!(
2564+
expr_from_projection(&projection[4]),
2565+
&Expr::Value(Value::DollarQuotedString(DollarQuotedString {
2566+
tag: None,
2567+
value: "".into()
2568+
})),
2569+
);
2570+
2571+
assert_eq!(
2572+
expr_from_projection(&projection[5]),
2573+
&Expr::Value(Value::DollarQuotedString(DollarQuotedString {
2574+
tag: Some("tag_name".into()),
2575+
value: "".into()
2576+
})),
2577+
);
2578+
}
2579+
2580+
#[test]
2581+
fn parse_incorrect_dollar_quoted_string() {
2582+
let sql = "SELECT $x$hello$$";
2583+
assert!(pg().parse_sql_statements(sql).is_err());
2584+
2585+
let sql = "SELECT $hello$$";
2586+
assert!(pg().parse_sql_statements(sql).is_err());
2587+
2588+
let sql = "SELECT $$$";
2589+
assert!(pg().parse_sql_statements(sql).is_err());
2590+
}

0 commit comments

Comments
 (0)