(datafusion-sqlparser-rs) branch main updated: SQLite: Support identifier characters above U+FFFF (#2555)

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-sqlparser-rs.git


The following commit(s) were added to refs/heads/main by this push:
     new 14cbf755 SQLite: Support identifier characters above U+FFFF (#2555)
14cbf755 is described below

commit 14cbf755b3f479a27f338c050217431279eb1f2e
Author: Luca Cappelletti <ca...@gmail.com>
AuthorDate: Fri Oct 2 19:32:23 2026 +0000

    SQLite: Support identifier characters above U+FFFF (#2555)
---
 src/dialect/sqlite.rs     |  5 +----
 tests/sqlparser_sqlite.rs | 13 +++++++++++++
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/dialect/sqlite.rs b/src/dialect/sqlite.rs
index 13bc555a..6f9ec0d2 100644
--- a/src/dialect/sqlite.rs
+++ b/src/dialect/sqlite.rs
@@ -48,10 +48,7 @@ impl Dialect for SQLiteDialect {
 
     fn is_identifier_start(&self, ch: char) -> bool {
         // See https://www.sqlite.org/draft/tokenreq.html
-        ch.is_ascii_lowercase()
-            || ch.is_ascii_uppercase()
-            || ch == '_'
-            || ('\u{007f}'..='\u{ffff}').contains(&ch)
+        ch.is_ascii_lowercase() || ch.is_ascii_uppercase() || ch == '_' || ch >= '\u{0080}'
     }
 
     fn supports_filter_during_aggregation(&self) -> bool {
diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs
index 81059761..b8fad7e9 100644
--- a/tests/sqlparser_sqlite.rs
+++ b/tests/sqlparser_sqlite.rs
@@ -974,6 +974,19 @@ fn parse_update_set_double_eq() {
     );
 }
 
+#[test]
+fn test_non_bmp_identifiers() {
+    // SQLite tokenizer treats every byte >= 0x80 as an identifier character,
+    // so any Unicode code point above U+007F is a valid identifier start/part.
+    sqlite().verified_stmt("SELECT 󟿾");
+    sqlite().verified_stmt("SELECT 𒀀");
+    sqlite().verified_stmt("SELECT 𒀀𒀁");
+    // U+DFFFE is not alphabetic, so GenericDialect rejects it as an identifier.
+    assert!(sqlparser::parser::Parser::parse_sql(&GenericDialect {}, "SELECT 󟿾").is_err());
+    // SQLite rejects U+007F as an unrecognized token.
+    assert!(sqlite().parse_sql_statements("SELECT \u{007f}").is_err());
+}
+
 #[test]
 fn parse_create_table_string_column_names() {
     sqlite().verified_stmt("CREATE TABLE t ('a')");


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org