You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/24 20:50:31 UTC
(datafusion-sqlparser-rs) 01/01: SQLite: Do not tokenize `N'...'` as a national string literal (#2561)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-2561-220e50a053b7ae225a9c08c246a3bbc1c4a1616c
in repository https://gitbox.apache.org/repos/asf/datafusion-sqlparser-rs.git
commit f72439961a6cfa55fd85980180d3aaa1daafccfa
Author: Luca Cappelletti <ca...@gmail.com>
AuthorDate: Thu Sep 24 20:50:06 2026 +0000
SQLite: Do not tokenize `N'...'` as a national string literal (#2561)
---
src/dialect/mod.rs | 8 ++++++++
src/dialect/sqlite.rs | 4 ++++
src/tokenizer.rs | 23 +++++++++++++----------
tests/sqlparser_common.rs | 13 ++++++++-----
tests/sqlparser_sqlite.rs | 12 ++++++++++++
5 files changed, 45 insertions(+), 15 deletions(-)
diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs
index 80d5c5811..4d9112f19 100644
--- a/src/dialect/mod.rs
+++ b/src/dialect/mod.rs
@@ -283,6 +283,14 @@ pub trait Dialect: Debug + Any {
false
}
+ /// Does the dialect tokenize `N'...'` as a national string literal?
+ ///
+ /// Dialects such as SQLite treat `N` as a plain identifier, so `N'foo'` is
+ /// the identifier `N` followed by a string literal, not a national string.
+ fn supports_national_string_literal(&self) -> bool {
+ true
+ }
+
/// Determine whether the dialect strips the backslash when escaping LIKE wildcards (%, _).
///
/// [MySQL] has a special case when escaping single quoted strings which leaves these unescaped
diff --git a/src/dialect/sqlite.rs b/src/dialect/sqlite.rs
index d549c7507..418a89e7c 100644
--- a/src/dialect/sqlite.rs
+++ b/src/dialect/sqlite.rs
@@ -129,4 +129,8 @@ impl Dialect for SQLiteDialect {
fn supports_numeric_literal_underscores(&self) -> bool {
true
}
+
+ fn supports_national_string_literal(&self) -> bool {
+ false
+ }
}
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index e17433e5d..7d363793c 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -1146,7 +1146,7 @@ impl<'a> Tokenizer<'a> {
n @ 'N' | n @ 'n' => {
chars.next(); // consume, to check the next char
match chars.peek() {
- Some('\'') => {
+ Some('\'') if self.dialect.supports_national_string_literal() => {
// N'...' - a <national character string literal>
let backslash_escape =
self.dialect.supports_string_literal_backslash_escape();
@@ -4125,15 +4125,18 @@ mod tests {
#[test]
fn test_national_strings_backslash_escape_not_supported() {
- all_dialects_where(|dialect| !dialect.supports_string_literal_backslash_escape())
- .tokenizes_to(
- "select n'''''\\'",
- vec![
- Token::make_keyword("select"),
- Token::Whitespace(Whitespace::Space),
- Token::NationalStringLiteral("''\\".to_string()),
- ],
- );
+ all_dialects_where(|dialect| {
+ !dialect.supports_string_literal_backslash_escape()
+ && dialect.supports_national_string_literal()
+ })
+ .tokenizes_to(
+ "select n'''''\\'",
+ vec![
+ Token::make_keyword("select"),
+ Token::Whitespace(Whitespace::Space),
+ Token::NationalStringLiteral("''\\".to_string()),
+ ],
+ );
}
#[test]
diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs
index 63639b916..449217b54 100644
--- a/tests/sqlparser_common.rs
+++ b/tests/sqlparser_common.rs
@@ -3803,7 +3803,8 @@ fn parse_listagg() {
verified_stmt("SELECT LISTAGG(dateid)");
verified_stmt("SELECT LISTAGG(DISTINCT dateid)");
verified_stmt("SELECT LISTAGG(dateid ON OVERFLOW ERROR)");
- verified_stmt("SELECT LISTAGG(dateid ON OVERFLOW TRUNCATE N'...' WITH COUNT)");
+ all_dialects_where(|d| d.supports_national_string_literal())
+ .verified_stmt("SELECT LISTAGG(dateid ON OVERFLOW TRUNCATE N'...' WITH COUNT)");
verified_stmt("SELECT LISTAGG(dateid ON OVERFLOW TRUNCATE X'deadbeef' WITH COUNT)");
}
@@ -6452,8 +6453,9 @@ fn parse_literal_decimal() {
#[test]
fn parse_literal_string() {
+ let national_string_dialects = all_dialects_where(|d| d.supports_national_string_literal());
let sql = "SELECT 'one', N'national string', X'deadBEEF'";
- let select = verified_only_select(sql);
+ let select = national_string_dialects.verified_only_select(sql);
assert_eq!(3, select.projection.len());
assert_eq!(
&Expr::Value((Value::SingleQuotedString("one".to_string())).with_empty_span()),
@@ -6470,9 +6472,10 @@ fn parse_literal_string() {
expr_from_projection(&select.projection[2])
);
- one_statement_parses_to("SELECT x'deadBEEF'", "SELECT X'deadBEEF'");
- one_statement_parses_to("SELECT n'national string'", "SELECT N'national string'");
- one_statement_parses_to(
+ all_dialects().one_statement_parses_to("SELECT x'deadBEEF'", "SELECT X'deadBEEF'");
+ national_string_dialects
+ .one_statement_parses_to("SELECT n'national string'", "SELECT N'national string'");
+ national_string_dialects.one_statement_parses_to(
r#"SELECT n'Tu geres '';'' et ''"'' ?'"#,
r#"SELECT N'Tu geres '';'' et ''"'' ?'"#,
);
diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs
index a017562df..22043004a 100644
--- a/tests/sqlparser_sqlite.rs
+++ b/tests/sqlparser_sqlite.rs
@@ -957,6 +957,18 @@ fn parse_pattern_operators_bind_at_like_precedence() {
}
}
+#[test]
+fn parse_n_prefix_not_national_string() {
+ // In SQLite, `n'...'` is the identifier `n` followed by a string literal.
+ // The string becomes an implicit alias, so `t.n''` round-trips as `t.n AS ''`.
+ sqlite().one_statement_parses_to("SELECT t.n'' FROM t", "SELECT t.n AS '' FROM t");
+ sqlite().one_statement_parses_to("SELECT n'' FROM t", "SELECT n AS '' FROM t");
+ sqlite().one_statement_parses_to("SELECT N'hello'", "SELECT N AS 'hello'");
+
+ // Other dialects still tokenize N'...' as a national string literal.
+ all_dialects_where(|d| d.supports_national_string_literal()).verified_stmt("SELECT N'hello'");
+}
+
fn sqlite() -> TestedDialects {
TestedDialects::new(vec![Box::new(SQLiteDialect {})])
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org