You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/01/13 10:24:56 UTC

(datafusion-sqlparser-rs) branch main updated: Tokenize empty line comments correctly (#2161)

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-sqlparser-rs.git


The following commit(s) were added to refs/heads/main by this push:
     new ee3b6223 Tokenize empty line comments correctly (#2161)
ee3b6223 is described below

commit ee3b6223e673b9c0f737d649a7f934de0186bc64
Author: Louis Vialar <zy...@users.noreply.github.com>
AuthorDate: Tue Jan 13 11:18:32 2026 +0100

    Tokenize empty line comments correctly (#2161)
---
 src/tokenizer.rs | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index eb935a4f..a9f9fb44 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -1387,7 +1387,11 @@ impl<'a> Tokenizer<'a> {
                         Some('-') => {
                             let mut is_comment = true;
                             if self.dialect.requires_single_line_comment_whitespace() {
-                                is_comment = Some(' ') == chars.peekable.clone().nth(1);
+                                is_comment = chars
+                                    .peekable
+                                    .clone()
+                                    .nth(1)
+                                    .is_some_and(char::is_whitespace);
                             }
 
                             if is_comment {
@@ -4069,6 +4073,24 @@ mod tests {
                     Token::Minus,
                 ],
             );
+
+        all_dialects_where(|d| d.requires_single_line_comment_whitespace()).tokenizes_to(
+            "--\n-- Table structure for table...\n--\n",
+            vec![
+                Token::Whitespace(Whitespace::SingleLineComment {
+                    prefix: "--".to_string(),
+                    comment: "\n".to_string(),
+                }),
+                Token::Whitespace(Whitespace::SingleLineComment {
+                    prefix: "--".to_string(),
+                    comment: " Table structure for table...\n".to_string(),
+                }),
+                Token::Whitespace(Whitespace::SingleLineComment {
+                    prefix: "--".to_string(),
+                    comment: "\n".to_string(),
+                }),
+            ],
+        );
     }
 
     #[test]


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org