You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2025/12/18 12:07:20 UTC
(datafusion-sqlparser-rs) 01/01: PostgreSQL Tokenization: Fix unexpected characters after question mark being silently ignored (#2129)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch gh-readonly-queue/main/pr-2129-d78dbc97a1f65ce7ee353fea181a6c3bb15a5050
in repository https://gitbox.apache.org/repos/asf/datafusion-sqlparser-rs.git
commit 39418cfebbfe0c028d780614e8b60ef8f0c98ce7
Author: jnlt3 <ds...@gmail.com>
AuthorDate: Thu Dec 18 15:06:55 2025 +0300
PostgreSQL Tokenization: Fix unexpected characters after question mark being silently ignored (#2129)
---
src/tokenizer.rs | 21 ++++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 2ae17cf4..8666563a 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -1717,7 +1717,7 @@ impl<'a> Tokenizer<'a> {
}
}
Some('#') => self.consume_and_return(chars, Token::QuestionMarkSharp),
- _ => self.consume_and_return(chars, Token::Question),
+ _ => Ok(Some(Token::Question)),
}
}
'?' => {
@@ -4147,4 +4147,23 @@ mod tests {
panic!("Tokenizer should have failed on {sql}, but it succeeded with {tokens:?}");
}
}
+
+ #[test]
+ fn tokenize_question_mark() {
+ let dialect = PostgreSqlDialect {};
+ let sql = "SELECT x ? y";
+ let tokens = Tokenizer::new(&dialect, sql).tokenize().unwrap();
+ compare(
+ tokens,
+ vec![
+ Token::make_keyword("SELECT"),
+ Token::Whitespace(Whitespace::Space),
+ Token::make_word("x", None),
+ Token::Whitespace(Whitespace::Space),
+ Token::Question,
+ Token::Whitespace(Whitespace::Space),
+ Token::make_word("y", None),
+ ],
+ )
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org