You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@paimon.apache.org by "XuQianJin-Stars (via GitHub)" <gi...@apache.org> on 2024/08/06 09:49:08 UTC
[PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
XuQianJin-Stars opened a new pull request, #42:
URL: https://github.com/apache/paimon-rust/pull/42
link: https://github.com/apache/paimon-rust/issues/33
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #42:
URL: https://github.com/apache/paimon-rust/pull/42#discussion_r1705255435
##########
crates/paimon/src/spec/types.rs:
##########
@@ -346,10 +429,13 @@ impl BooleanType {
/// Data type of a fixed-length character string.
///
/// Impl Reference: <https://github.com/apache/paimon/blob/release-0.8.2/paimon-common/src/main/java/org/apache/paimon/types/CharType.java>.
+#[serde_as]
#[derive(Debug, Clone, PartialEq, Hash, Eq, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct CharType {
+ #[serde_as(as = "DisplayFromStr")]
Review Comment:
Hi, I believe this is wrong since `CharType` should be serialized into `CHAR(n) NOT NULL`.
The `DisplayFromStr` should be added upon `CharType` instead of field instead. Also, `CharType` should implement `FromStr` which can parse a `CharType` from given string `CHAR(n) NOT NULL`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "Aitozi (via GitHub)" <gi...@apache.org>.
Aitozi commented on code in PR #42:
URL: https://github.com/apache/paimon-rust/pull/42#discussion_r1708948350
##########
crates/paimon/src/spec/types.rs:
##########
@@ -1338,4 +1496,204 @@ mod tests {
));
assert_eq!(row_type.to_string(), "ROW<INTEGER, ARRAY<INTEGER>>");
}
+
+ #[test]
+ fn test_datatype_serialize_deserialize() {
+ // BooleanType
+ let bool_type: BooleanType = BooleanType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&bool_type).unwrap();
+ assert_eq!(serialized, json!("BOOLEAN").to_string());
+ let bool_type: BooleanType = BooleanType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&bool_type).unwrap();
+ assert_eq!(serialized, json!("BOOLEAN NOT NULL").to_string());
+
+ // TinyIntType
+ let tiny_type: TinyIntType = TinyIntType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&tiny_type).unwrap();
+ assert_eq!(serialized, json!("TINYINT").to_string());
+ let tiny_type: TinyIntType = TinyIntType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&tiny_type).unwrap();
+ assert_eq!(serialized, json!("TINYINT NOT NULL").to_string());
+
+ // SmallIntType
+ let small_type: SmallIntType = SmallIntType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&small_type).unwrap();
+ assert_eq!(serialized, json!("SMALLINT").to_string());
+ let small_type: SmallIntType = SmallIntType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&small_type).unwrap();
+ assert_eq!(serialized, json!("SMALLINT NOT NULL").to_string());
+
+ // IntType
+ let int_type: IntType = IntType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&int_type).unwrap();
+ assert_eq!(serialized, json!("INTEGER").to_string());
+ let int_type: IntType = IntType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&int_type).unwrap();
+ assert_eq!(serialized, json!("INTEGER NOT NULL").to_string());
+
+ // BigIntType
+ let big_type: BigIntType = BigIntType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&big_type).unwrap();
+ assert_eq!(serialized, json!("BIGINT").to_string());
+ let big_type: BigIntType = BigIntType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&big_type).unwrap();
+ assert_eq!(serialized, json!("BIGINT NOT NULL").to_string());
+
+ // DecimalType
+ let decimal_type: DecimalType = DecimalType::with_nullable(true, 10, 2).unwrap();
+ let serialized: String = serde_json::to_string(&decimal_type).unwrap();
+ assert_eq!(serialized, json!("DECIMAL(10, 2)").to_string());
+ let decimal_type: DecimalType = DecimalType::with_nullable(false, 10, 2).unwrap();
+ let serialized: String = serde_json::to_string(&decimal_type).unwrap();
+ assert_eq!(serialized, json!("DECIMAL(10, 2) NOT NULL").to_string());
+
+ // DoubleType
+ let double_type: DoubleType = DoubleType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&double_type).unwrap();
+ assert_eq!(serialized, json!("DOUBLE").to_string());
+ let double_type: DoubleType = DoubleType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&double_type).unwrap();
+ assert_eq!(serialized, json!("DOUBLE NOT NULL").to_string());
+
+ // FloatType
+ let float_type: FloatType = FloatType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&float_type).unwrap();
+ assert_eq!(serialized, json!("FLOAT").to_string());
+ let float_type: FloatType = FloatType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&float_type).unwrap();
+ assert_eq!(serialized, json!("FLOAT NOT NULL").to_string());
+
+ // BinaryType
+ let binary_type: BinaryType = BinaryType::with_nullable(true, 10).unwrap();
+ let serialized: String = serde_json::to_string(&binary_type).unwrap();
+ assert_eq!(serialized, json!("BINARY(10)").to_string());
+ let binary_type: BinaryType = BinaryType::with_nullable(false, 10).unwrap();
+ let serialized: String = serde_json::to_string(&binary_type).unwrap();
+ assert_eq!(serialized, json!("BINARY(10) NOT NULL").to_string());
+
+ // VarBinaryType
+ let varbinary_type: VarBinaryType = VarBinaryType::try_new(true, 10).unwrap();
+ let serialized: String = serde_json::to_string(&varbinary_type).unwrap();
+ assert_eq!(serialized, json!("VARBINARY(10)").to_string());
+ let varbinary_type: VarBinaryType = VarBinaryType::try_new(false, 10).unwrap();
+ let serialized: String = serde_json::to_string(&varbinary_type).unwrap();
+ assert_eq!(serialized, json!("VARBINARY(10) NOT NULL").to_string());
+
+ // CharType
+ let char_type: CharType = CharType::with_nullable(true, 10).unwrap();
+ let serialized: String = serde_json::to_string(&char_type).unwrap();
+ assert_eq!(serialized, json!("CHAR(10)").to_string());
+ let char_type: CharType = CharType::with_nullable(false, 10).unwrap();
+ let serialized: String = serde_json::to_string(&char_type).unwrap();
+ assert_eq!(serialized, json!("CHAR(10) NOT NULL").to_string());
+
+ // VarCharType
+ let varchar_type: VarCharType = VarCharType::with_nullable(true, 10).unwrap();
+ let serialized: String = serde_json::to_string(&varchar_type).unwrap();
+ assert_eq!(serialized, json!("VARCHAR(10)").to_string());
+ let varchar_type: VarCharType = VarCharType::with_nullable(false, 10).unwrap();
+ let serialized: String = serde_json::to_string(&varchar_type).unwrap();
+ assert_eq!(serialized, json!("VARCHAR(10) NOT NULL").to_string());
+
+ // DateType
+ let date_type: DateType = DateType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&date_type).unwrap();
+ assert_eq!(serialized, json!("DATE").to_string());
+ let date_type: DateType = DateType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&date_type).unwrap();
+ assert_eq!(serialized, json!("DATE NOT NULL").to_string());
+
+ // LocalZonedTimestampType
+ let localzoned_type: LocalZonedTimestampType =
+ LocalZonedTimestampType::with_nullable(true, 6).unwrap();
+ let serialized: String = serde_json::to_string(&localzoned_type).unwrap();
+ assert_eq!(
+ serialized,
+ json!("TIMESTAMP WITH LOCAL TIME ZONE(6)").to_string()
+ );
+ let localzoned_type: LocalZonedTimestampType =
+ LocalZonedTimestampType::with_nullable(false, 6).unwrap();
+ let serialized: String = serde_json::to_string(&localzoned_type).unwrap();
+ assert_eq!(
+ serialized,
+ json!("TIMESTAMP WITH LOCAL TIME ZONE(6) NOT NULL").to_string()
+ );
+
+ // TimeType
+ let time_type: TimeType = TimeType::with_nullable(true, 6).unwrap();
+ let serialized: String = serde_json::to_string(&time_type).unwrap();
+ assert_eq!(serialized, json!("TIME(6)").to_string());
+ let time_type: TimeType = TimeType::with_nullable(false, 6).unwrap();
+ let serialized: String = serde_json::to_string(&time_type).unwrap();
+ assert_eq!(serialized, json!("TIME(6) NOT NULL").to_string());
+ let time_type: TimeType = TimeType::with_nullable(false, 6).unwrap();
+ let serialized: String = serde_json::to_string(&time_type).unwrap();
+ assert_eq!(serialized, json!("TIME(6) NOT NULL").to_string());
+
+ // TimestampType
+ let timestamp_type: TimestampType = TimestampType::with_nullable(true, 6).unwrap();
+ let serialized: String = serde_json::to_string(×tamp_type).unwrap();
+ assert_eq!(serialized, json!("TIMESTAMP(6)").to_string());
+ let timestamp_type: TimestampType = TimestampType::with_nullable(false, 6).unwrap();
+ let serialized: String = serde_json::to_string(×tamp_type).unwrap();
+ assert_eq!(serialized, json!("TIMESTAMP(6) NOT NULL").to_string());
+
+ // ArrayType
+ let int_type: DataType = DataType::Int(IntType::with_nullable(true));
+ let array_type: ArrayType = ArrayType::with_nullable(true, int_type.clone());
+ let serialized: String = serde_json::to_string(&array_type).unwrap();
+ assert_eq!(serialized, json!("ARRAY<INTEGER>").to_string());
+ let array_type: ArrayType = ArrayType::with_nullable(false, int_type.clone());
+ let serialized: String = serde_json::to_string(&array_type).unwrap();
+ assert_eq!(serialized, json!("ARRAY<INTEGER> NOT NULL").to_string());
Review Comment:
This serialized json is not right. These four types have different representation from the `SerializeDisplay`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on PR #42:
URL: https://github.com/apache/paimon-rust/pull/42#issuecomment-2280771599
Thank you very much for your effort. I believe we can implement this in a more rusty way. I presented my idea here: https://github.com/apache/paimon-rust/pull/48. Would you like to take a look?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "Aitozi (via GitHub)" <gi...@apache.org>.
Aitozi commented on code in PR #42:
URL: https://github.com/apache/paimon-rust/pull/42#discussion_r1709456058
##########
crates/paimon/src/spec/types.rs:
##########
@@ -1338,4 +1496,204 @@ mod tests {
));
assert_eq!(row_type.to_string(), "ROW<INTEGER, ARRAY<INTEGER>>");
}
+
+ #[test]
+ fn test_datatype_serialize_deserialize() {
+ // BooleanType
+ let bool_type: BooleanType = BooleanType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&bool_type).unwrap();
+ assert_eq!(serialized, json!("BOOLEAN").to_string());
+ let bool_type: BooleanType = BooleanType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&bool_type).unwrap();
+ assert_eq!(serialized, json!("BOOLEAN NOT NULL").to_string());
+
+ // TinyIntType
+ let tiny_type: TinyIntType = TinyIntType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&tiny_type).unwrap();
+ assert_eq!(serialized, json!("TINYINT").to_string());
+ let tiny_type: TinyIntType = TinyIntType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&tiny_type).unwrap();
+ assert_eq!(serialized, json!("TINYINT NOT NULL").to_string());
+
+ // SmallIntType
+ let small_type: SmallIntType = SmallIntType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&small_type).unwrap();
+ assert_eq!(serialized, json!("SMALLINT").to_string());
+ let small_type: SmallIntType = SmallIntType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&small_type).unwrap();
+ assert_eq!(serialized, json!("SMALLINT NOT NULL").to_string());
+
+ // IntType
+ let int_type: IntType = IntType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&int_type).unwrap();
+ assert_eq!(serialized, json!("INTEGER").to_string());
+ let int_type: IntType = IntType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&int_type).unwrap();
+ assert_eq!(serialized, json!("INTEGER NOT NULL").to_string());
+
+ // BigIntType
+ let big_type: BigIntType = BigIntType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&big_type).unwrap();
+ assert_eq!(serialized, json!("BIGINT").to_string());
+ let big_type: BigIntType = BigIntType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&big_type).unwrap();
+ assert_eq!(serialized, json!("BIGINT NOT NULL").to_string());
+
+ // DecimalType
+ let decimal_type: DecimalType = DecimalType::with_nullable(true, 10, 2).unwrap();
+ let serialized: String = serde_json::to_string(&decimal_type).unwrap();
+ assert_eq!(serialized, json!("DECIMAL(10, 2)").to_string());
+ let decimal_type: DecimalType = DecimalType::with_nullable(false, 10, 2).unwrap();
+ let serialized: String = serde_json::to_string(&decimal_type).unwrap();
+ assert_eq!(serialized, json!("DECIMAL(10, 2) NOT NULL").to_string());
+
+ // DoubleType
+ let double_type: DoubleType = DoubleType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&double_type).unwrap();
+ assert_eq!(serialized, json!("DOUBLE").to_string());
+ let double_type: DoubleType = DoubleType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&double_type).unwrap();
+ assert_eq!(serialized, json!("DOUBLE NOT NULL").to_string());
+
+ // FloatType
+ let float_type: FloatType = FloatType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&float_type).unwrap();
+ assert_eq!(serialized, json!("FLOAT").to_string());
+ let float_type: FloatType = FloatType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&float_type).unwrap();
+ assert_eq!(serialized, json!("FLOAT NOT NULL").to_string());
+
+ // BinaryType
+ let binary_type: BinaryType = BinaryType::with_nullable(true, 10).unwrap();
+ let serialized: String = serde_json::to_string(&binary_type).unwrap();
+ assert_eq!(serialized, json!("BINARY(10)").to_string());
+ let binary_type: BinaryType = BinaryType::with_nullable(false, 10).unwrap();
+ let serialized: String = serde_json::to_string(&binary_type).unwrap();
+ assert_eq!(serialized, json!("BINARY(10) NOT NULL").to_string());
+
+ // VarBinaryType
+ let varbinary_type: VarBinaryType = VarBinaryType::try_new(true, 10).unwrap();
+ let serialized: String = serde_json::to_string(&varbinary_type).unwrap();
+ assert_eq!(serialized, json!("VARBINARY(10)").to_string());
+ let varbinary_type: VarBinaryType = VarBinaryType::try_new(false, 10).unwrap();
+ let serialized: String = serde_json::to_string(&varbinary_type).unwrap();
+ assert_eq!(serialized, json!("VARBINARY(10) NOT NULL").to_string());
+
+ // CharType
+ let char_type: CharType = CharType::with_nullable(true, 10).unwrap();
+ let serialized: String = serde_json::to_string(&char_type).unwrap();
+ assert_eq!(serialized, json!("CHAR(10)").to_string());
+ let char_type: CharType = CharType::with_nullable(false, 10).unwrap();
+ let serialized: String = serde_json::to_string(&char_type).unwrap();
+ assert_eq!(serialized, json!("CHAR(10) NOT NULL").to_string());
+
+ // VarCharType
+ let varchar_type: VarCharType = VarCharType::with_nullable(true, 10).unwrap();
+ let serialized: String = serde_json::to_string(&varchar_type).unwrap();
+ assert_eq!(serialized, json!("VARCHAR(10)").to_string());
+ let varchar_type: VarCharType = VarCharType::with_nullable(false, 10).unwrap();
+ let serialized: String = serde_json::to_string(&varchar_type).unwrap();
+ assert_eq!(serialized, json!("VARCHAR(10) NOT NULL").to_string());
+
+ // DateType
+ let date_type: DateType = DateType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&date_type).unwrap();
+ assert_eq!(serialized, json!("DATE").to_string());
+ let date_type: DateType = DateType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&date_type).unwrap();
+ assert_eq!(serialized, json!("DATE NOT NULL").to_string());
+
+ // LocalZonedTimestampType
+ let localzoned_type: LocalZonedTimestampType =
+ LocalZonedTimestampType::with_nullable(true, 6).unwrap();
+ let serialized: String = serde_json::to_string(&localzoned_type).unwrap();
+ assert_eq!(
+ serialized,
+ json!("TIMESTAMP WITH LOCAL TIME ZONE(6)").to_string()
+ );
+ let localzoned_type: LocalZonedTimestampType =
+ LocalZonedTimestampType::with_nullable(false, 6).unwrap();
+ let serialized: String = serde_json::to_string(&localzoned_type).unwrap();
+ assert_eq!(
+ serialized,
+ json!("TIMESTAMP WITH LOCAL TIME ZONE(6) NOT NULL").to_string()
+ );
+
+ // TimeType
+ let time_type: TimeType = TimeType::with_nullable(true, 6).unwrap();
+ let serialized: String = serde_json::to_string(&time_type).unwrap();
+ assert_eq!(serialized, json!("TIME(6)").to_string());
+ let time_type: TimeType = TimeType::with_nullable(false, 6).unwrap();
+ let serialized: String = serde_json::to_string(&time_type).unwrap();
+ assert_eq!(serialized, json!("TIME(6) NOT NULL").to_string());
+ let time_type: TimeType = TimeType::with_nullable(false, 6).unwrap();
+ let serialized: String = serde_json::to_string(&time_type).unwrap();
+ assert_eq!(serialized, json!("TIME(6) NOT NULL").to_string());
+
+ // TimestampType
+ let timestamp_type: TimestampType = TimestampType::with_nullable(true, 6).unwrap();
+ let serialized: String = serde_json::to_string(×tamp_type).unwrap();
+ assert_eq!(serialized, json!("TIMESTAMP(6)").to_string());
+ let timestamp_type: TimestampType = TimestampType::with_nullable(false, 6).unwrap();
+ let serialized: String = serde_json::to_string(×tamp_type).unwrap();
+ assert_eq!(serialized, json!("TIMESTAMP(6) NOT NULL").to_string());
+
+ // ArrayType
+ let int_type: DataType = DataType::Int(IntType::with_nullable(true));
+ let array_type: ArrayType = ArrayType::with_nullable(true, int_type.clone());
+ let serialized: String = serde_json::to_string(&array_type).unwrap();
+ assert_eq!(serialized, json!("ARRAY<INTEGER>").to_string());
+ let array_type: ArrayType = ArrayType::with_nullable(false, int_type.clone());
+ let serialized: String = serde_json::to_string(&array_type).unwrap();
+ assert_eq!(serialized, json!("ARRAY<INTEGER> NOT NULL").to_string());
Review Comment:
<img width="364" alt="image" src="https://github.com/user-attachments/assets/3ff73fa3-ad96-4fe3-911d-364c5863a90d">
The serialized version for `ARRAY<STRING>`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "XuQianJin-Stars (via GitHub)" <gi...@apache.org>.
XuQianJin-Stars commented on PR #42:
URL: https://github.com/apache/paimon-rust/pull/42#issuecomment-2273511889
This pull request still has `Deserialize` and some optimizations to be completed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "devillove084 (via GitHub)" <gi...@apache.org>.
devillove084 commented on code in PR #42:
URL: https://github.com/apache/paimon-rust/pull/42#discussion_r1712613087
##########
crates/paimon/src/spec/tokens.rs:
##########
@@ -0,0 +1,1246 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::error::DataTypeParsingSnafu;
+use crate::spec::types::DataType;
+use crate::Error;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::{collections::HashSet, fmt, str::FromStr};
+
+use super::{
+ ArrayType, BigIntType, BinaryType, BooleanType, CharType, DataField, DateType, DecimalType,
+ DoubleType, FloatType, IntType, LocalZonedTimestampType, MapType, MultisetType, RowType,
+ SmallIntType, TimeType, TimestampType, TinyIntType, VarBinaryType, VarCharType,
+};
+
+/// --------------------------------------------------------------------------------------------
+/// Tokenizer
+/// --------------------------------------------------------------------------------------------
+
+const CHAR_BEGIN_SUBTYPE: char = '<';
+const CHAR_END_SUBTYPE: char = '>';
+const CHAR_BEGIN_PARAMETER: char = '(';
+const CHAR_END_PARAMETER: char = ')';
+const CHAR_LIST_SEPARATOR: char = ',';
+const CHAR_STRING: char = '\'';
+const CHAR_IDENTIFIER: char = '`';
+const CHAR_DOT: char = '.';
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)]
+pub enum Keyword {
+ CHAR,
+ VARCHAR,
+ STRING,
+ BOOLEAN,
+ BINARY,
+ VARBINARY,
+ BYTES,
+ DECIMAL,
+ NUMERIC,
+ DEC,
+ TINYINT,
+ SMALLINT,
+ INT,
+ INTEGER,
+ BIGINT,
+ FLOAT,
+ DOUBLE,
+ PRECISION,
+ DATE,
+ TIME,
+ WITH,
+ WITHOUT,
+ LOCAL,
+ ZONE,
+ TIMESTAMP,
+ TimestampLtz,
+ INTERVAL,
+ YEAR,
+ MONTH,
+ DAY,
+ HOUR,
+ MINUTE,
+ SECOND,
+ TO,
+ ARRAY,
+ MULTISET,
+ MAP,
+ ROW,
+ NULL,
+ RAW,
+ LEGACY,
+ NOT,
+}
+
+impl fmt::Display for Keyword {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ Keyword::CHAR => write!(f, "CHAR"),
+ Keyword::VARCHAR => write!(f, "VARCHAR"),
+ Keyword::STRING => write!(f, "STRING"),
+ Keyword::BOOLEAN => write!(f, "BOOLEAN"),
+ Keyword::BINARY => write!(f, "BINARY"),
+ Keyword::VARBINARY => write!(f, "VARBINARY"),
+ Keyword::BYTES => write!(f, "BYTES"),
+ Keyword::DECIMAL => write!(f, "DECIMAL"),
+ Keyword::NUMERIC => write!(f, "NUMERIC"),
+ Keyword::DEC => write!(f, "DEC"),
+ Keyword::TINYINT => write!(f, "TINYINT"),
+ Keyword::SMALLINT => write!(f, "SMALLINT"),
+ Keyword::INT => write!(f, "INT"),
+ Keyword::INTEGER => write!(f, "INTEGER"),
+ Keyword::BIGINT => write!(f, "BIGINT"),
+ Keyword::FLOAT => write!(f, "FLOAT"),
+ Keyword::DOUBLE => write!(f, "DOUBLE"),
+ Keyword::PRECISION => write!(f, "PRECISION"),
+ Keyword::DATE => write!(f, "DATE"),
+ Keyword::TIME => write!(f, "TIME"),
+ Keyword::WITH => write!(f, "WITH"),
+ Keyword::WITHOUT => write!(f, "WITHOUT"),
+ Keyword::LOCAL => write!(f, "LOCAL"),
+ Keyword::ZONE => write!(f, "ZONE"),
+ Keyword::TIMESTAMP => write!(f, "TIMESTAMP"),
+ Keyword::TimestampLtz => write!(f, "TIMESTAMP_LTZ"),
+ Keyword::INTERVAL => write!(f, "INTERVAL"),
+ Keyword::YEAR => write!(f, "YEAR"),
+ Keyword::MONTH => write!(f, "MONTH"),
+ Keyword::DAY => write!(f, "DAY"),
+ Keyword::HOUR => write!(f, "HOUR"),
+ Keyword::MINUTE => write!(f, "MINUTE"),
+ Keyword::SECOND => write!(f, "SECOND"),
+ Keyword::TO => write!(f, "TO"),
+ Keyword::ARRAY => write!(f, "ARRAY"),
+ Keyword::MULTISET => write!(f, "MULTISET"),
+ Keyword::MAP => write!(f, "MAP"),
+ Keyword::ROW => write!(f, "ROW"),
+ Keyword::NULL => write!(f, "NULL"),
+ Keyword::RAW => write!(f, "RAW"),
+ Keyword::LEGACY => write!(f, "LEGACY"),
+ Keyword::NOT => write!(f, "NOT"),
+ }
+ }
+}
+
+impl FromStr for Keyword {
+ type Err = ();
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s.to_uppercase().as_str() {
+ "CHAR" => Ok(Keyword::CHAR),
+ "VARCHAR" => Ok(Keyword::VARCHAR),
+ "STRING" => Ok(Keyword::STRING),
+ "BOOLEAN" => Ok(Keyword::BOOLEAN),
+ "BINARY" => Ok(Keyword::BINARY),
+ "VARBINARY" => Ok(Keyword::VARBINARY),
+ "BYTES" => Ok(Keyword::BYTES),
+ "DECIMAL" => Ok(Keyword::DECIMAL),
+ "NUMERIC" => Ok(Keyword::NUMERIC),
+ "DEC" => Ok(Keyword::DEC),
+ "TINYINT" => Ok(Keyword::TINYINT),
+ "SMALLINT" => Ok(Keyword::SMALLINT),
+ "INT" => Ok(Keyword::INT),
+ "INTEGER" => Ok(Keyword::INTEGER),
+ "BIGINT" => Ok(Keyword::BIGINT),
+ "FLOAT" => Ok(Keyword::FLOAT),
+ "DOUBLE" => Ok(Keyword::DOUBLE),
+ "PRECISION" => Ok(Keyword::PRECISION),
+ "DATE" => Ok(Keyword::DATE),
+ "TIME" => Ok(Keyword::TIME),
+ "WITH" => Ok(Keyword::WITH),
+ "WITHOUT" => Ok(Keyword::WITHOUT),
+ "LOCAL" => Ok(Keyword::LOCAL),
+ "ZONE" => Ok(Keyword::ZONE),
+ "TIMESTAMP" => Ok(Keyword::TIMESTAMP),
+ "TIMESTAMP_LTZ" => Ok(Keyword::TimestampLtz),
+ "INTERVAL" => Ok(Keyword::INTERVAL),
+ "YEAR" => Ok(Keyword::YEAR),
+ "MONTH" => Ok(Keyword::MONTH),
+ "DAY" => Ok(Keyword::DAY),
+ "HOUR" => Ok(Keyword::HOUR),
+ "MINUTE" => Ok(Keyword::MINUTE),
+ "SECOND" => Ok(Keyword::SECOND),
+ "TO" => Ok(Keyword::TO),
+ "ARRAY" => Ok(Keyword::ARRAY),
+ "MULTISET" => Ok(Keyword::MULTISET),
+ "MAP" => Ok(Keyword::MAP),
+ "ROW" => Ok(Keyword::ROW),
+ "NULL" => Ok(Keyword::NULL),
+ "RAW" => Ok(Keyword::RAW),
+ "LEGACY" => Ok(Keyword::LEGACY),
+ "NOT" => Ok(Keyword::NOT),
+ _ => Err(()),
+ }
+ }
+}
+
+impl Keyword {
+ pub fn variants() -> HashSet<Keyword> {
+ vec![
+ Keyword::CHAR,
+ Keyword::VARCHAR,
+ Keyword::STRING,
+ Keyword::BOOLEAN,
+ Keyword::BINARY,
+ Keyword::VARBINARY,
+ Keyword::BYTES,
+ Keyword::DECIMAL,
+ Keyword::NUMERIC,
+ Keyword::DEC,
+ Keyword::TINYINT,
+ Keyword::SMALLINT,
+ Keyword::INT,
+ Keyword::INTEGER,
+ Keyword::BIGINT,
+ Keyword::FLOAT,
+ Keyword::DOUBLE,
+ Keyword::PRECISION,
+ Keyword::DATE,
+ Keyword::TIME,
+ Keyword::WITH,
+ Keyword::WITHOUT,
+ Keyword::LOCAL,
+ Keyword::ZONE,
+ Keyword::TIMESTAMP,
+ Keyword::TimestampLtz,
+ Keyword::INTERVAL,
+ Keyword::YEAR,
+ Keyword::MONTH,
+ Keyword::DAY,
+ Keyword::HOUR,
+ Keyword::MINUTE,
+ Keyword::SECOND,
+ Keyword::TO,
+ Keyword::ARRAY,
+ Keyword::MULTISET,
+ Keyword::MAP,
+ Keyword::ROW,
+ Keyword::NULL,
+ Keyword::RAW,
+ Keyword::LEGACY,
+ Keyword::NOT,
+ ]
+ .into_iter()
+ .collect()
+ }
+
+ pub fn is_keyword(s: &str) -> bool {
+ Keyword::variants().iter().any(|v| v.to_string() == s)
+ }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum TokenType {
+ /// e.g. "ROW<"
+ BeginSubtype,
+ /// e.g. "ROW<..>"
+ EndSubtype,
+ /// e.g. "CHAR("
+ BeginParameter,
+ /// e.g. "CHAR(...)"
+ EndParameter,
+ /// e.g. "ROW<INT,"
+ ListSeparator,
+ /// e.g. "ROW<name INT 'Comment'"
+ LiteralString,
+ /// CHAR(12
+ LiteralInt,
+ /// e.g. "CHAR" or "TO"
+ Keyword,
+ /// e.g. "ROW<name" or "myCatalog.myDatabase"
+ Identifier,
+ /// e.g. "myCatalog.myDatabase."
+ IdentifierSeparator,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Token {
+ token_type: TokenType,
+ cursor_position: isize,
+ value: String,
+}
+
+impl Token {
+ pub fn new(token_type: TokenType, cursor_position: isize, value: String) -> Self {
+ Token {
+ token_type,
+ cursor_position,
+ value,
+ }
+ }
+}
+
+pub struct TokenParser {
+ input_string: String,
+ tokens: Vec<Token>,
+ last_valid_token: isize,
+ current_token: isize,
+}
+
+impl TokenParser {
+ pub fn new(input_string: String, tokens: Vec<Token>) -> Self {
+ TokenParser {
+ input_string,
+ tokens,
+ last_valid_token: -1,
+ current_token: -1,
+ }
+ }
+
+ pub fn parse_tokens(&mut self) -> Result<DataType, Error> {
+ let type_data = self.parse_type_with_nullability();
+ if self.has_remaining_tokens() {
+ self.next_token();
+ self.parsing_error(format!("Unexpected token: {}", self.token().value));
+ }
+ Ok(type_data)
+ }
+
+ pub fn last_cursor(&self) -> usize {
+ if self.last_valid_token < 0 {
+ 0
+ } else {
+ self.tokens[self.last_valid_token as usize].cursor_position as usize + 1
+ }
+ }
+
+ pub fn parsing_error(&self, cause: String) -> ! {
+ panic!(
+ "Could not parse type at position {}: {}\nInput type string: {}",
+ self.last_cursor(),
+ cause,
+ self.input_string
+ );
+ }
+
+ pub fn has_remaining_tokens(&self) -> bool {
+ self.current_token + 1 < self.tokens.len() as isize
+ }
+
+ pub fn token(&self) -> &Token {
+ &self.tokens[self.current_token as usize]
+ }
+
+ pub fn token_as_int(&self) -> isize {
+ self.token().value.parse::<isize>().unwrap_or_else(|_| {
+ self.parsing_error(String::from("Invalid integer value."));
+ })
+ }
+
+ pub fn token_as_keyword(&self) -> Keyword {
Review Comment:
Here you can directly use the Keyword::from_str method instead of repeatedly defining the matching logic. Something like:
``` rust
Keyword::from_str(self.token().value.as_str())
.unwrap_or_else(|_| self.parsing_error(format!("Expected keyword, found: {}", self.token().value)))
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #42:
URL: https://github.com/apache/paimon-rust/pull/42#discussion_r1709464476
##########
crates/paimon/src/spec/types.rs:
##########
@@ -1338,4 +1496,204 @@ mod tests {
));
assert_eq!(row_type.to_string(), "ROW<INTEGER, ARRAY<INTEGER>>");
}
+
+ #[test]
+ fn test_datatype_serialize_deserialize() {
+ // BooleanType
+ let bool_type: BooleanType = BooleanType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&bool_type).unwrap();
+ assert_eq!(serialized, json!("BOOLEAN").to_string());
+ let bool_type: BooleanType = BooleanType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&bool_type).unwrap();
+ assert_eq!(serialized, json!("BOOLEAN NOT NULL").to_string());
+
+ // TinyIntType
+ let tiny_type: TinyIntType = TinyIntType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&tiny_type).unwrap();
+ assert_eq!(serialized, json!("TINYINT").to_string());
+ let tiny_type: TinyIntType = TinyIntType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&tiny_type).unwrap();
+ assert_eq!(serialized, json!("TINYINT NOT NULL").to_string());
+
+ // SmallIntType
+ let small_type: SmallIntType = SmallIntType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&small_type).unwrap();
+ assert_eq!(serialized, json!("SMALLINT").to_string());
+ let small_type: SmallIntType = SmallIntType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&small_type).unwrap();
+ assert_eq!(serialized, json!("SMALLINT NOT NULL").to_string());
+
+ // IntType
+ let int_type: IntType = IntType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&int_type).unwrap();
+ assert_eq!(serialized, json!("INTEGER").to_string());
+ let int_type: IntType = IntType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&int_type).unwrap();
+ assert_eq!(serialized, json!("INTEGER NOT NULL").to_string());
+
+ // BigIntType
+ let big_type: BigIntType = BigIntType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&big_type).unwrap();
+ assert_eq!(serialized, json!("BIGINT").to_string());
+ let big_type: BigIntType = BigIntType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&big_type).unwrap();
+ assert_eq!(serialized, json!("BIGINT NOT NULL").to_string());
+
+ // DecimalType
+ let decimal_type: DecimalType = DecimalType::with_nullable(true, 10, 2).unwrap();
+ let serialized: String = serde_json::to_string(&decimal_type).unwrap();
+ assert_eq!(serialized, json!("DECIMAL(10, 2)").to_string());
+ let decimal_type: DecimalType = DecimalType::with_nullable(false, 10, 2).unwrap();
+ let serialized: String = serde_json::to_string(&decimal_type).unwrap();
+ assert_eq!(serialized, json!("DECIMAL(10, 2) NOT NULL").to_string());
+
+ // DoubleType
+ let double_type: DoubleType = DoubleType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&double_type).unwrap();
+ assert_eq!(serialized, json!("DOUBLE").to_string());
+ let double_type: DoubleType = DoubleType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&double_type).unwrap();
+ assert_eq!(serialized, json!("DOUBLE NOT NULL").to_string());
+
+ // FloatType
+ let float_type: FloatType = FloatType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&float_type).unwrap();
+ assert_eq!(serialized, json!("FLOAT").to_string());
+ let float_type: FloatType = FloatType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&float_type).unwrap();
+ assert_eq!(serialized, json!("FLOAT NOT NULL").to_string());
+
+ // BinaryType
+ let binary_type: BinaryType = BinaryType::with_nullable(true, 10).unwrap();
+ let serialized: String = serde_json::to_string(&binary_type).unwrap();
+ assert_eq!(serialized, json!("BINARY(10)").to_string());
+ let binary_type: BinaryType = BinaryType::with_nullable(false, 10).unwrap();
+ let serialized: String = serde_json::to_string(&binary_type).unwrap();
+ assert_eq!(serialized, json!("BINARY(10) NOT NULL").to_string());
+
+ // VarBinaryType
+ let varbinary_type: VarBinaryType = VarBinaryType::try_new(true, 10).unwrap();
+ let serialized: String = serde_json::to_string(&varbinary_type).unwrap();
+ assert_eq!(serialized, json!("VARBINARY(10)").to_string());
+ let varbinary_type: VarBinaryType = VarBinaryType::try_new(false, 10).unwrap();
+ let serialized: String = serde_json::to_string(&varbinary_type).unwrap();
+ assert_eq!(serialized, json!("VARBINARY(10) NOT NULL").to_string());
+
+ // CharType
+ let char_type: CharType = CharType::with_nullable(true, 10).unwrap();
+ let serialized: String = serde_json::to_string(&char_type).unwrap();
+ assert_eq!(serialized, json!("CHAR(10)").to_string());
+ let char_type: CharType = CharType::with_nullable(false, 10).unwrap();
+ let serialized: String = serde_json::to_string(&char_type).unwrap();
+ assert_eq!(serialized, json!("CHAR(10) NOT NULL").to_string());
+
+ // VarCharType
+ let varchar_type: VarCharType = VarCharType::with_nullable(true, 10).unwrap();
+ let serialized: String = serde_json::to_string(&varchar_type).unwrap();
+ assert_eq!(serialized, json!("VARCHAR(10)").to_string());
+ let varchar_type: VarCharType = VarCharType::with_nullable(false, 10).unwrap();
+ let serialized: String = serde_json::to_string(&varchar_type).unwrap();
+ assert_eq!(serialized, json!("VARCHAR(10) NOT NULL").to_string());
+
+ // DateType
+ let date_type: DateType = DateType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&date_type).unwrap();
+ assert_eq!(serialized, json!("DATE").to_string());
+ let date_type: DateType = DateType::with_nullable(false);
+ let serialized: String = serde_json::to_string(&date_type).unwrap();
+ assert_eq!(serialized, json!("DATE NOT NULL").to_string());
+
+ // LocalZonedTimestampType
+ let localzoned_type: LocalZonedTimestampType =
+ LocalZonedTimestampType::with_nullable(true, 6).unwrap();
+ let serialized: String = serde_json::to_string(&localzoned_type).unwrap();
+ assert_eq!(
+ serialized,
+ json!("TIMESTAMP WITH LOCAL TIME ZONE(6)").to_string()
+ );
+ let localzoned_type: LocalZonedTimestampType =
+ LocalZonedTimestampType::with_nullable(false, 6).unwrap();
+ let serialized: String = serde_json::to_string(&localzoned_type).unwrap();
+ assert_eq!(
+ serialized,
+ json!("TIMESTAMP WITH LOCAL TIME ZONE(6) NOT NULL").to_string()
+ );
+
+ // TimeType
+ let time_type: TimeType = TimeType::with_nullable(true, 6).unwrap();
+ let serialized: String = serde_json::to_string(&time_type).unwrap();
+ assert_eq!(serialized, json!("TIME(6)").to_string());
+ let time_type: TimeType = TimeType::with_nullable(false, 6).unwrap();
+ let serialized: String = serde_json::to_string(&time_type).unwrap();
+ assert_eq!(serialized, json!("TIME(6) NOT NULL").to_string());
+ let time_type: TimeType = TimeType::with_nullable(false, 6).unwrap();
+ let serialized: String = serde_json::to_string(&time_type).unwrap();
+ assert_eq!(serialized, json!("TIME(6) NOT NULL").to_string());
+
+ // TimestampType
+ let timestamp_type: TimestampType = TimestampType::with_nullable(true, 6).unwrap();
+ let serialized: String = serde_json::to_string(×tamp_type).unwrap();
+ assert_eq!(serialized, json!("TIMESTAMP(6)").to_string());
+ let timestamp_type: TimestampType = TimestampType::with_nullable(false, 6).unwrap();
+ let serialized: String = serde_json::to_string(×tamp_type).unwrap();
+ assert_eq!(serialized, json!("TIMESTAMP(6) NOT NULL").to_string());
+
+ // ArrayType
+ let int_type: DataType = DataType::Int(IntType::with_nullable(true));
+ let array_type: ArrayType = ArrayType::with_nullable(true, int_type.clone());
+ let serialized: String = serde_json::to_string(&array_type).unwrap();
+ assert_eq!(serialized, json!("ARRAY<INTEGER>").to_string());
+ let array_type: ArrayType = ArrayType::with_nullable(false, int_type.clone());
+ let serialized: String = serde_json::to_string(&array_type).unwrap();
+ assert_eq!(serialized, json!("ARRAY<INTEGER> NOT NULL").to_string());
Review Comment:
I'm a bit confused. @JingsongLi mentioned that Paimon will use `as_sql_string` everywhere. I'll double-check this part.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "devillove084 (via GitHub)" <gi...@apache.org>.
devillove084 commented on code in PR #42:
URL: https://github.com/apache/paimon-rust/pull/42#discussion_r1712614051
##########
crates/paimon/src/spec/tokens.rs:
##########
@@ -0,0 +1,1246 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::error::DataTypeParsingSnafu;
+use crate::spec::types::DataType;
+use crate::Error;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::{collections::HashSet, fmt, str::FromStr};
+
+use super::{
+ ArrayType, BigIntType, BinaryType, BooleanType, CharType, DataField, DateType, DecimalType,
+ DoubleType, FloatType, IntType, LocalZonedTimestampType, MapType, MultisetType, RowType,
+ SmallIntType, TimeType, TimestampType, TinyIntType, VarBinaryType, VarCharType,
+};
+
+/// --------------------------------------------------------------------------------------------
+/// Tokenizer
+/// --------------------------------------------------------------------------------------------
+
+const CHAR_BEGIN_SUBTYPE: char = '<';
+const CHAR_END_SUBTYPE: char = '>';
+const CHAR_BEGIN_PARAMETER: char = '(';
+const CHAR_END_PARAMETER: char = ')';
+const CHAR_LIST_SEPARATOR: char = ',';
+const CHAR_STRING: char = '\'';
+const CHAR_IDENTIFIER: char = '`';
+const CHAR_DOT: char = '.';
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)]
+pub enum Keyword {
+ CHAR,
+ VARCHAR,
+ STRING,
+ BOOLEAN,
+ BINARY,
+ VARBINARY,
+ BYTES,
+ DECIMAL,
+ NUMERIC,
+ DEC,
+ TINYINT,
+ SMALLINT,
+ INT,
+ INTEGER,
+ BIGINT,
+ FLOAT,
+ DOUBLE,
+ PRECISION,
+ DATE,
+ TIME,
+ WITH,
+ WITHOUT,
+ LOCAL,
+ ZONE,
+ TIMESTAMP,
+ TimestampLtz,
+ INTERVAL,
+ YEAR,
+ MONTH,
+ DAY,
+ HOUR,
+ MINUTE,
+ SECOND,
+ TO,
+ ARRAY,
+ MULTISET,
+ MAP,
+ ROW,
+ NULL,
+ RAW,
+ LEGACY,
+ NOT,
+}
+
+impl fmt::Display for Keyword {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ Keyword::CHAR => write!(f, "CHAR"),
+ Keyword::VARCHAR => write!(f, "VARCHAR"),
+ Keyword::STRING => write!(f, "STRING"),
+ Keyword::BOOLEAN => write!(f, "BOOLEAN"),
+ Keyword::BINARY => write!(f, "BINARY"),
+ Keyword::VARBINARY => write!(f, "VARBINARY"),
+ Keyword::BYTES => write!(f, "BYTES"),
+ Keyword::DECIMAL => write!(f, "DECIMAL"),
+ Keyword::NUMERIC => write!(f, "NUMERIC"),
+ Keyword::DEC => write!(f, "DEC"),
+ Keyword::TINYINT => write!(f, "TINYINT"),
+ Keyword::SMALLINT => write!(f, "SMALLINT"),
+ Keyword::INT => write!(f, "INT"),
+ Keyword::INTEGER => write!(f, "INTEGER"),
+ Keyword::BIGINT => write!(f, "BIGINT"),
+ Keyword::FLOAT => write!(f, "FLOAT"),
+ Keyword::DOUBLE => write!(f, "DOUBLE"),
+ Keyword::PRECISION => write!(f, "PRECISION"),
+ Keyword::DATE => write!(f, "DATE"),
+ Keyword::TIME => write!(f, "TIME"),
+ Keyword::WITH => write!(f, "WITH"),
+ Keyword::WITHOUT => write!(f, "WITHOUT"),
+ Keyword::LOCAL => write!(f, "LOCAL"),
+ Keyword::ZONE => write!(f, "ZONE"),
+ Keyword::TIMESTAMP => write!(f, "TIMESTAMP"),
+ Keyword::TimestampLtz => write!(f, "TIMESTAMP_LTZ"),
+ Keyword::INTERVAL => write!(f, "INTERVAL"),
+ Keyword::YEAR => write!(f, "YEAR"),
+ Keyword::MONTH => write!(f, "MONTH"),
+ Keyword::DAY => write!(f, "DAY"),
+ Keyword::HOUR => write!(f, "HOUR"),
+ Keyword::MINUTE => write!(f, "MINUTE"),
+ Keyword::SECOND => write!(f, "SECOND"),
+ Keyword::TO => write!(f, "TO"),
+ Keyword::ARRAY => write!(f, "ARRAY"),
+ Keyword::MULTISET => write!(f, "MULTISET"),
+ Keyword::MAP => write!(f, "MAP"),
+ Keyword::ROW => write!(f, "ROW"),
+ Keyword::NULL => write!(f, "NULL"),
+ Keyword::RAW => write!(f, "RAW"),
+ Keyword::LEGACY => write!(f, "LEGACY"),
+ Keyword::NOT => write!(f, "NOT"),
+ }
+ }
+}
+
+impl FromStr for Keyword {
+ type Err = ();
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s.to_uppercase().as_str() {
+ "CHAR" => Ok(Keyword::CHAR),
+ "VARCHAR" => Ok(Keyword::VARCHAR),
+ "STRING" => Ok(Keyword::STRING),
+ "BOOLEAN" => Ok(Keyword::BOOLEAN),
+ "BINARY" => Ok(Keyword::BINARY),
+ "VARBINARY" => Ok(Keyword::VARBINARY),
+ "BYTES" => Ok(Keyword::BYTES),
+ "DECIMAL" => Ok(Keyword::DECIMAL),
+ "NUMERIC" => Ok(Keyword::NUMERIC),
+ "DEC" => Ok(Keyword::DEC),
+ "TINYINT" => Ok(Keyword::TINYINT),
+ "SMALLINT" => Ok(Keyword::SMALLINT),
+ "INT" => Ok(Keyword::INT),
+ "INTEGER" => Ok(Keyword::INTEGER),
+ "BIGINT" => Ok(Keyword::BIGINT),
+ "FLOAT" => Ok(Keyword::FLOAT),
+ "DOUBLE" => Ok(Keyword::DOUBLE),
+ "PRECISION" => Ok(Keyword::PRECISION),
+ "DATE" => Ok(Keyword::DATE),
+ "TIME" => Ok(Keyword::TIME),
+ "WITH" => Ok(Keyword::WITH),
+ "WITHOUT" => Ok(Keyword::WITHOUT),
+ "LOCAL" => Ok(Keyword::LOCAL),
+ "ZONE" => Ok(Keyword::ZONE),
+ "TIMESTAMP" => Ok(Keyword::TIMESTAMP),
+ "TIMESTAMP_LTZ" => Ok(Keyword::TimestampLtz),
+ "INTERVAL" => Ok(Keyword::INTERVAL),
+ "YEAR" => Ok(Keyword::YEAR),
+ "MONTH" => Ok(Keyword::MONTH),
+ "DAY" => Ok(Keyword::DAY),
+ "HOUR" => Ok(Keyword::HOUR),
+ "MINUTE" => Ok(Keyword::MINUTE),
+ "SECOND" => Ok(Keyword::SECOND),
+ "TO" => Ok(Keyword::TO),
+ "ARRAY" => Ok(Keyword::ARRAY),
+ "MULTISET" => Ok(Keyword::MULTISET),
+ "MAP" => Ok(Keyword::MAP),
+ "ROW" => Ok(Keyword::ROW),
+ "NULL" => Ok(Keyword::NULL),
+ "RAW" => Ok(Keyword::RAW),
+ "LEGACY" => Ok(Keyword::LEGACY),
+ "NOT" => Ok(Keyword::NOT),
+ _ => Err(()),
+ }
+ }
+}
+
+impl Keyword {
+ pub fn variants() -> HashSet<Keyword> {
+ vec![
+ Keyword::CHAR,
+ Keyword::VARCHAR,
+ Keyword::STRING,
+ Keyword::BOOLEAN,
+ Keyword::BINARY,
+ Keyword::VARBINARY,
+ Keyword::BYTES,
+ Keyword::DECIMAL,
+ Keyword::NUMERIC,
+ Keyword::DEC,
+ Keyword::TINYINT,
+ Keyword::SMALLINT,
+ Keyword::INT,
+ Keyword::INTEGER,
+ Keyword::BIGINT,
+ Keyword::FLOAT,
+ Keyword::DOUBLE,
+ Keyword::PRECISION,
+ Keyword::DATE,
+ Keyword::TIME,
+ Keyword::WITH,
+ Keyword::WITHOUT,
+ Keyword::LOCAL,
+ Keyword::ZONE,
+ Keyword::TIMESTAMP,
+ Keyword::TimestampLtz,
+ Keyword::INTERVAL,
+ Keyword::YEAR,
+ Keyword::MONTH,
+ Keyword::DAY,
+ Keyword::HOUR,
+ Keyword::MINUTE,
+ Keyword::SECOND,
+ Keyword::TO,
+ Keyword::ARRAY,
+ Keyword::MULTISET,
+ Keyword::MAP,
+ Keyword::ROW,
+ Keyword::NULL,
+ Keyword::RAW,
+ Keyword::LEGACY,
+ Keyword::NOT,
+ ]
+ .into_iter()
+ .collect()
+ }
+
+ pub fn is_keyword(s: &str) -> bool {
+ Keyword::variants().iter().any(|v| v.to_string() == s)
+ }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum TokenType {
+ /// e.g. "ROW<"
+ BeginSubtype,
+ /// e.g. "ROW<..>"
+ EndSubtype,
+ /// e.g. "CHAR("
+ BeginParameter,
+ /// e.g. "CHAR(...)"
+ EndParameter,
+ /// e.g. "ROW<INT,"
+ ListSeparator,
+ /// e.g. "ROW<name INT 'Comment'"
+ LiteralString,
+ /// CHAR(12
+ LiteralInt,
+ /// e.g. "CHAR" or "TO"
+ Keyword,
+ /// e.g. "ROW<name" or "myCatalog.myDatabase"
+ Identifier,
+ /// e.g. "myCatalog.myDatabase."
+ IdentifierSeparator,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Token {
+ token_type: TokenType,
+ cursor_position: isize,
+ value: String,
+}
+
+impl Token {
+ pub fn new(token_type: TokenType, cursor_position: isize, value: String) -> Self {
+ Token {
+ token_type,
+ cursor_position,
+ value,
+ }
+ }
+}
+
+pub struct TokenParser {
+ input_string: String,
+ tokens: Vec<Token>,
+ last_valid_token: isize,
+ current_token: isize,
+}
+
+impl TokenParser {
+ pub fn new(input_string: String, tokens: Vec<Token>) -> Self {
+ TokenParser {
+ input_string,
+ tokens,
+ last_valid_token: -1,
+ current_token: -1,
+ }
+ }
+
+ pub fn parse_tokens(&mut self) -> Result<DataType, Error> {
+ let type_data = self.parse_type_with_nullability();
+ if self.has_remaining_tokens() {
+ self.next_token();
+ self.parsing_error(format!("Unexpected token: {}", self.token().value));
+ }
+ Ok(type_data)
+ }
+
+ pub fn last_cursor(&self) -> usize {
+ if self.last_valid_token < 0 {
+ 0
+ } else {
+ self.tokens[self.last_valid_token as usize].cursor_position as usize + 1
+ }
+ }
+
+ pub fn parsing_error(&self, cause: String) -> ! {
+ panic!(
Review Comment:
Using panic directly here might not be ideal, but we can leave a TODO to explore a better approach in the future.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #42:
URL: https://github.com/apache/paimon-rust/pull/42#discussion_r1707039850
##########
crates/paimon/src/spec/types.rs:
##########
@@ -124,6 +126,10 @@ impl DataType {
DataType::Row(v) => v.nullable,
}
}
+
+ fn serde_json(&self) -> serde_json::Value {
Review Comment:
Hi, I belive we don't need API like `serde_json`. Users will use `serde_json::to_string(&data_type)` directly
##########
crates/paimon/src/spec/types.rs:
##########
@@ -243,12 +294,54 @@ impl BigIntType {
}
}
+impl Serialize for BigIntType {
+ fn serialize<S>(
+ &self,
+ serializer: S,
+ ) -> Result<<S as serde::Serializer>::Ok, <S as serde::Serializer>::Error>
+ where
+ S: serde::Serializer,
+ {
+ serializer.serialize_str(&self.to_string())
+ }
+}
+
+impl<'de> Deserialize<'de> for BigIntType {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ deserializer.deserialize_str(BigIntTypeVisitor)
Review Comment:
I prefer to implement `from_str` instead and forward `Deserialize` to `from_str`.
##########
crates/paimon/src/spec/types.rs:
##########
@@ -200,12 +239,24 @@ impl ArrayType {
}
}
+impl Serialize for ArrayType {
+ fn serialize<S>(
+ &self,
+ serializer: S,
+ ) -> Result<<S as serde::Serializer>::Ok, <S as serde::Serializer>::Error>
+ where
+ S: serde::Serializer,
+ {
+ serializer.serialize_str(&self.to_string())
Review Comment:
Can we use `#[serde_as(as = "DisplayFromStr")]` instead of implement `Serialize` by hand?
##########
crates/paimon/src/spec/types.rs:
##########
@@ -157,7 +163,40 @@ impl FromStr for DataType {
type Err = Error;
fn from_str(_: &str) -> Result<Self, Self::Err> {
- todo!()
+ todo!();
+ }
+}
+
+impl Serialize for DataType {
Review Comment:
Hi, this has native support in serde. Please add `#[serde(untagged)]` for `DataType` instead.
Reference: https://serde.rs/enum-representations.html#untagged
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "devillove084 (via GitHub)" <gi...@apache.org>.
devillove084 commented on code in PR #42:
URL: https://github.com/apache/paimon-rust/pull/42#discussion_r1712613692
##########
crates/paimon/src/spec/tokens.rs:
##########
@@ -0,0 +1,1246 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::error::DataTypeParsingSnafu;
+use crate::spec::types::DataType;
+use crate::Error;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::{collections::HashSet, fmt, str::FromStr};
+
+use super::{
+ ArrayType, BigIntType, BinaryType, BooleanType, CharType, DataField, DateType, DecimalType,
+ DoubleType, FloatType, IntType, LocalZonedTimestampType, MapType, MultisetType, RowType,
+ SmallIntType, TimeType, TimestampType, TinyIntType, VarBinaryType, VarCharType,
+};
+
+/// --------------------------------------------------------------------------------------------
+/// Tokenizer
+/// --------------------------------------------------------------------------------------------
+
+const CHAR_BEGIN_SUBTYPE: char = '<';
+const CHAR_END_SUBTYPE: char = '>';
+const CHAR_BEGIN_PARAMETER: char = '(';
+const CHAR_END_PARAMETER: char = ')';
+const CHAR_LIST_SEPARATOR: char = ',';
+const CHAR_STRING: char = '\'';
+const CHAR_IDENTIFIER: char = '`';
+const CHAR_DOT: char = '.';
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)]
+pub enum Keyword {
+ CHAR,
+ VARCHAR,
+ STRING,
+ BOOLEAN,
+ BINARY,
+ VARBINARY,
+ BYTES,
+ DECIMAL,
+ NUMERIC,
+ DEC,
+ TINYINT,
+ SMALLINT,
+ INT,
+ INTEGER,
+ BIGINT,
+ FLOAT,
+ DOUBLE,
+ PRECISION,
+ DATE,
+ TIME,
+ WITH,
+ WITHOUT,
+ LOCAL,
+ ZONE,
+ TIMESTAMP,
+ TimestampLtz,
+ INTERVAL,
+ YEAR,
+ MONTH,
+ DAY,
+ HOUR,
+ MINUTE,
+ SECOND,
+ TO,
+ ARRAY,
+ MULTISET,
+ MAP,
+ ROW,
+ NULL,
+ RAW,
+ LEGACY,
+ NOT,
+}
+
+impl fmt::Display for Keyword {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ Keyword::CHAR => write!(f, "CHAR"),
+ Keyword::VARCHAR => write!(f, "VARCHAR"),
+ Keyword::STRING => write!(f, "STRING"),
+ Keyword::BOOLEAN => write!(f, "BOOLEAN"),
+ Keyword::BINARY => write!(f, "BINARY"),
+ Keyword::VARBINARY => write!(f, "VARBINARY"),
+ Keyword::BYTES => write!(f, "BYTES"),
+ Keyword::DECIMAL => write!(f, "DECIMAL"),
+ Keyword::NUMERIC => write!(f, "NUMERIC"),
+ Keyword::DEC => write!(f, "DEC"),
+ Keyword::TINYINT => write!(f, "TINYINT"),
+ Keyword::SMALLINT => write!(f, "SMALLINT"),
+ Keyword::INT => write!(f, "INT"),
+ Keyword::INTEGER => write!(f, "INTEGER"),
+ Keyword::BIGINT => write!(f, "BIGINT"),
+ Keyword::FLOAT => write!(f, "FLOAT"),
+ Keyword::DOUBLE => write!(f, "DOUBLE"),
+ Keyword::PRECISION => write!(f, "PRECISION"),
+ Keyword::DATE => write!(f, "DATE"),
+ Keyword::TIME => write!(f, "TIME"),
+ Keyword::WITH => write!(f, "WITH"),
+ Keyword::WITHOUT => write!(f, "WITHOUT"),
+ Keyword::LOCAL => write!(f, "LOCAL"),
+ Keyword::ZONE => write!(f, "ZONE"),
+ Keyword::TIMESTAMP => write!(f, "TIMESTAMP"),
+ Keyword::TimestampLtz => write!(f, "TIMESTAMP_LTZ"),
+ Keyword::INTERVAL => write!(f, "INTERVAL"),
+ Keyword::YEAR => write!(f, "YEAR"),
+ Keyword::MONTH => write!(f, "MONTH"),
+ Keyword::DAY => write!(f, "DAY"),
+ Keyword::HOUR => write!(f, "HOUR"),
+ Keyword::MINUTE => write!(f, "MINUTE"),
+ Keyword::SECOND => write!(f, "SECOND"),
+ Keyword::TO => write!(f, "TO"),
+ Keyword::ARRAY => write!(f, "ARRAY"),
+ Keyword::MULTISET => write!(f, "MULTISET"),
+ Keyword::MAP => write!(f, "MAP"),
+ Keyword::ROW => write!(f, "ROW"),
+ Keyword::NULL => write!(f, "NULL"),
+ Keyword::RAW => write!(f, "RAW"),
+ Keyword::LEGACY => write!(f, "LEGACY"),
+ Keyword::NOT => write!(f, "NOT"),
+ }
+ }
+}
+
+impl FromStr for Keyword {
+ type Err = ();
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s.to_uppercase().as_str() {
+ "CHAR" => Ok(Keyword::CHAR),
+ "VARCHAR" => Ok(Keyword::VARCHAR),
+ "STRING" => Ok(Keyword::STRING),
+ "BOOLEAN" => Ok(Keyword::BOOLEAN),
+ "BINARY" => Ok(Keyword::BINARY),
+ "VARBINARY" => Ok(Keyword::VARBINARY),
+ "BYTES" => Ok(Keyword::BYTES),
+ "DECIMAL" => Ok(Keyword::DECIMAL),
+ "NUMERIC" => Ok(Keyword::NUMERIC),
+ "DEC" => Ok(Keyword::DEC),
+ "TINYINT" => Ok(Keyword::TINYINT),
+ "SMALLINT" => Ok(Keyword::SMALLINT),
+ "INT" => Ok(Keyword::INT),
+ "INTEGER" => Ok(Keyword::INTEGER),
+ "BIGINT" => Ok(Keyword::BIGINT),
+ "FLOAT" => Ok(Keyword::FLOAT),
+ "DOUBLE" => Ok(Keyword::DOUBLE),
+ "PRECISION" => Ok(Keyword::PRECISION),
+ "DATE" => Ok(Keyword::DATE),
+ "TIME" => Ok(Keyword::TIME),
+ "WITH" => Ok(Keyword::WITH),
+ "WITHOUT" => Ok(Keyword::WITHOUT),
+ "LOCAL" => Ok(Keyword::LOCAL),
+ "ZONE" => Ok(Keyword::ZONE),
+ "TIMESTAMP" => Ok(Keyword::TIMESTAMP),
+ "TIMESTAMP_LTZ" => Ok(Keyword::TimestampLtz),
+ "INTERVAL" => Ok(Keyword::INTERVAL),
+ "YEAR" => Ok(Keyword::YEAR),
+ "MONTH" => Ok(Keyword::MONTH),
+ "DAY" => Ok(Keyword::DAY),
+ "HOUR" => Ok(Keyword::HOUR),
+ "MINUTE" => Ok(Keyword::MINUTE),
+ "SECOND" => Ok(Keyword::SECOND),
+ "TO" => Ok(Keyword::TO),
+ "ARRAY" => Ok(Keyword::ARRAY),
+ "MULTISET" => Ok(Keyword::MULTISET),
+ "MAP" => Ok(Keyword::MAP),
+ "ROW" => Ok(Keyword::ROW),
+ "NULL" => Ok(Keyword::NULL),
+ "RAW" => Ok(Keyword::RAW),
+ "LEGACY" => Ok(Keyword::LEGACY),
+ "NOT" => Ok(Keyword::NOT),
+ _ => Err(()),
+ }
+ }
+}
+
+impl Keyword {
+ pub fn variants() -> HashSet<Keyword> {
+ vec![
+ Keyword::CHAR,
+ Keyword::VARCHAR,
+ Keyword::STRING,
+ Keyword::BOOLEAN,
+ Keyword::BINARY,
+ Keyword::VARBINARY,
+ Keyword::BYTES,
+ Keyword::DECIMAL,
+ Keyword::NUMERIC,
+ Keyword::DEC,
+ Keyword::TINYINT,
+ Keyword::SMALLINT,
+ Keyword::INT,
+ Keyword::INTEGER,
+ Keyword::BIGINT,
+ Keyword::FLOAT,
+ Keyword::DOUBLE,
+ Keyword::PRECISION,
+ Keyword::DATE,
+ Keyword::TIME,
+ Keyword::WITH,
+ Keyword::WITHOUT,
+ Keyword::LOCAL,
+ Keyword::ZONE,
+ Keyword::TIMESTAMP,
+ Keyword::TimestampLtz,
+ Keyword::INTERVAL,
+ Keyword::YEAR,
+ Keyword::MONTH,
+ Keyword::DAY,
+ Keyword::HOUR,
+ Keyword::MINUTE,
+ Keyword::SECOND,
+ Keyword::TO,
+ Keyword::ARRAY,
+ Keyword::MULTISET,
+ Keyword::MAP,
+ Keyword::ROW,
+ Keyword::NULL,
+ Keyword::RAW,
+ Keyword::LEGACY,
+ Keyword::NOT,
+ ]
+ .into_iter()
+ .collect()
+ }
+
+ pub fn is_keyword(s: &str) -> bool {
+ Keyword::variants().iter().any(|v| v.to_string() == s)
+ }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum TokenType {
+ /// e.g. "ROW<"
+ BeginSubtype,
+ /// e.g. "ROW<..>"
+ EndSubtype,
+ /// e.g. "CHAR("
+ BeginParameter,
+ /// e.g. "CHAR(...)"
+ EndParameter,
+ /// e.g. "ROW<INT,"
+ ListSeparator,
+ /// e.g. "ROW<name INT 'Comment'"
+ LiteralString,
+ /// CHAR(12
+ LiteralInt,
+ /// e.g. "CHAR" or "TO"
+ Keyword,
+ /// e.g. "ROW<name" or "myCatalog.myDatabase"
+ Identifier,
+ /// e.g. "myCatalog.myDatabase."
+ IdentifierSeparator,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Token {
+ token_type: TokenType,
+ cursor_position: isize,
+ value: String,
+}
+
+impl Token {
+ pub fn new(token_type: TokenType, cursor_position: isize, value: String) -> Self {
+ Token {
+ token_type,
+ cursor_position,
+ value,
+ }
+ }
+}
+
+pub struct TokenParser {
+ input_string: String,
+ tokens: Vec<Token>,
+ last_valid_token: isize,
+ current_token: isize,
+}
+
+impl TokenParser {
+ pub fn new(input_string: String, tokens: Vec<Token>) -> Self {
+ TokenParser {
+ input_string,
+ tokens,
+ last_valid_token: -1,
+ current_token: -1,
+ }
+ }
+
+ pub fn parse_tokens(&mut self) -> Result<DataType, Error> {
+ let type_data = self.parse_type_with_nullability();
+ if self.has_remaining_tokens() {
+ self.next_token();
+ self.parsing_error(format!("Unexpected token: {}", self.token().value));
+ }
+ Ok(type_data)
+ }
+
+ pub fn last_cursor(&self) -> usize {
+ if self.last_valid_token < 0 {
+ 0
+ } else {
+ self.tokens[self.last_valid_token as usize].cursor_position as usize + 1
+ }
+ }
+
+ pub fn parsing_error(&self, cause: String) -> ! {
+ panic!(
+ "Could not parse type at position {}: {}\nInput type string: {}",
+ self.last_cursor(),
+ cause,
+ self.input_string
+ );
+ }
+
+ pub fn has_remaining_tokens(&self) -> bool {
+ self.current_token + 1 < self.tokens.len() as isize
+ }
+
+ pub fn token(&self) -> &Token {
+ &self.tokens[self.current_token as usize]
+ }
+
+ pub fn token_as_int(&self) -> isize {
+ self.token().value.parse::<isize>().unwrap_or_else(|_| {
+ self.parsing_error(String::from("Invalid integer value."));
+ })
+ }
+
+ pub fn token_as_keyword(&self) -> Keyword {
+ if Keyword::is_keyword(self.token().value.as_str()) {
+ match self.token().value.as_str() {
+ "CHAR" => Keyword::CHAR,
+ "VARCHAR" => Keyword::VARCHAR,
+ "STRING" => Keyword::STRING,
+ "BOOLEAN" => Keyword::BOOLEAN,
+ "BINARY" => Keyword::BINARY,
+ "VARBINARY" => Keyword::VARBINARY,
+ "BYTES" => Keyword::BYTES,
+ "DECIMAL" => Keyword::DECIMAL,
+ "NUMERIC" => Keyword::NUMERIC,
+ "DEC" => Keyword::DEC,
+ "TINYINT" => Keyword::TINYINT,
+ "SMALLINT" => Keyword::SMALLINT,
+ "INT" => Keyword::INT,
+ "INTEGER" => Keyword::INTEGER,
+ "BIGINT" => Keyword::BIGINT,
+ "FLOAT" => Keyword::FLOAT,
+ "DOUBLE" => Keyword::DOUBLE,
+ "PRECISION" => Keyword::PRECISION,
+ "DATE" => Keyword::DATE,
+ "TIME" => Keyword::TIME,
+ "WITH" => Keyword::WITH,
+ "WITHOUT" => Keyword::WITHOUT,
+ "LOCAL" => Keyword::LOCAL,
+ "ZONE" => Keyword::ZONE,
+ "TIMESTAMP" => Keyword::TIMESTAMP,
+ "TIMESTAMP_LTZ" => Keyword::TimestampLtz,
+ "INTERVAL" => Keyword::INTERVAL,
+ "YEAR" => Keyword::YEAR,
+ "MONTH" => Keyword::MONTH,
+ "DAY" => Keyword::DAY,
+ "HOUR" => Keyword::HOUR,
+ "MINUTE" => Keyword::MINUTE,
+ "SECOND" => Keyword::SECOND,
+ "TO" => Keyword::TO,
+ "ARRAY" => Keyword::ARRAY,
+ "MULTISET" => Keyword::MULTISET,
+ "MAP" => Keyword::MAP,
+ "ROW" => Keyword::ROW,
+ "NULL" => Keyword::NULL,
+ "RAW" => Keyword::RAW,
+ "LEGACY" => Keyword::LEGACY,
+ "NOT" => Keyword::NOT,
+ _ => panic!("Unsupported type: {}", self.token().value),
+ }
+ } else {
+ self.parsing_error(format!("Expected keyword, found: {}", self.token().value));
+ }
+ }
+
+ pub fn next_token(&mut self) {
+ self.current_token += 1;
+ if self.current_token >= self.tokens.len() as isize {
+ self.parsing_error(String::from("Unexpected end."));
+ } else {
+ self.last_valid_token = self.current_token - 1;
+ }
+ }
+
+ pub fn next_token_type(&mut self, expected_type: TokenType) {
+ self.next_token();
+ let token = self.token();
+ if token.token_type != expected_type {
+ self.parsing_error(format!(
+ "<{:?}> expected but was <{:?}>.",
+ expected_type, token.token_type
+ ));
+ }
+ }
+
+ pub fn next_token_keyword(&mut self, expected_keyword: Keyword) {
+ self.next_token();
+ let token = self.token();
+ if token.token_type != TokenType::Keyword {
+ self.parsing_error(format!(
+ "<{:?}> expected but was <{:?}>.",
+ expected_keyword, token.token_type
+ ));
+ }
+ }
+
+ pub fn has_next_token_type(&mut self, types: Vec<TokenType>) -> bool {
+ if self.current_token + types.len() as isize + 1 > self.tokens.len() as isize {
+ return false;
+ }
+ for (i, type_) in types.iter().enumerate() {
+ let look_ahead = &self.tokens[self.current_token as usize + i + 1];
+ if look_ahead.token_type != *type_ {
+ return false;
+ }
+ }
+ true
+ }
+
+ pub fn has_next_token_keyword(&mut self, keywords: Vec<Keyword>) -> bool {
+ if self.current_token + keywords.len() as isize + 1 > self.tokens.len() as isize {
+ return false;
+ }
+ for (i, v) in keywords.iter().enumerate() {
+ let look_ahead = &self.tokens[self.current_token as usize + i + 1];
+ if look_ahead.token_type != TokenType::Keyword
+ || *v != Keyword::from_str(look_ahead.value.as_str()).unwrap()
+ {
+ return false;
+ }
+ }
+ true
+ }
+
+ pub fn parse_nullability(&mut self) -> bool {
+ if self.has_next_token_keyword(vec![Keyword::NOT, Keyword::NULL]) {
+ self.next_token_keyword(Keyword::NOT);
+ self.next_token_keyword(Keyword::NULL);
+ false
+ } else if self.has_next_token_keyword(vec![Keyword::NULL]) {
+ self.next_token_keyword(Keyword::NULL);
+ true
+ } else {
+ true
+ }
+ }
+
+ pub fn parse_type_with_nullability(&mut self) -> DataType {
+ let data_type = self
+ .parse_type_by_keyword()
+ .unwrap()
+ .copy(self.parse_nullability());
+
+ if self.has_next_token_keyword(vec![Keyword::ARRAY]) {
+ self.next_token_keyword(Keyword::ARRAY);
+ return DataType::Array(ArrayType::with_nullable(
+ self.parse_nullability(),
+ data_type,
+ ));
+ } else if self.has_next_token_keyword(vec![Keyword::MULTISET]) {
+ self.next_token_keyword(Keyword::MULTISET);
+ return DataType::Multiset(MultisetType::with_nullable(
+ self.parse_nullability(),
+ data_type,
+ ));
+ }
+
+ data_type
+ }
+
+ pub fn parse_type_by_keyword(&mut self) -> Result<DataType, Error> {
+ self.next_token_type(TokenType::Keyword);
+ match self.token_as_keyword() {
+ Keyword::CHAR => Ok(self.parse_char_type()),
+ Keyword::VARCHAR => Ok(self.parse_var_char_type()),
+ Keyword::STRING => Ok(DataType::VarChar(VarCharType::default())),
+ Keyword::BOOLEAN => Ok(DataType::Boolean(BooleanType::new())),
+ Keyword::BINARY => Ok(self.parse_binary_type()),
+ Keyword::VARBINARY => Ok(self.parse_var_binary_type()),
+ Keyword::BYTES => Ok(DataType::VarBinary(
+ VarBinaryType::new(VarBinaryType::MAX_LENGTH).unwrap(),
+ )),
+ Keyword::DECIMAL | Keyword::NUMERIC | Keyword::DEC => Ok(self.parse_decimal_type()),
+ Keyword::TINYINT => Ok(DataType::TinyInt(TinyIntType::default())),
+ Keyword::SMALLINT => Ok(DataType::SmallInt(SmallIntType::default())),
+ Keyword::INT | Keyword::INTEGER => Ok(DataType::Int(IntType::default())),
+ Keyword::BIGINT => Ok(DataType::BigInt(BigIntType::default())),
+ Keyword::FLOAT => Ok(DataType::Float(FloatType::new())),
+ Keyword::DOUBLE => Ok(self.parse_double_type()),
+ Keyword::DATE => Ok(DataType::Date(DateType::new())),
+ Keyword::TIME => Ok(self.parse_time_type()),
+ Keyword::TIMESTAMP => Ok(self.parse_timestamp_type()),
+ Keyword::TimestampLtz => Ok(self.parse_timestamp_ltz_type()),
+ _ => DataTypeParsingSnafu {
+ message: format!("Unsupported type: {}", self.token().value),
+ }
+ .fail(),
+ }
+ }
+
+ pub fn parse_string_type(&mut self) -> isize {
+ if self.has_next_token_type(vec![TokenType::BeginParameter]) {
+ self.next_token_type(TokenType::BeginParameter);
+ self.next_token_type(TokenType::LiteralInt);
+ let length = self.token_as_int();
+ self.next_token_type(TokenType::EndParameter);
+ length
+ } else {
+ -1
+ }
+ }
+
+ pub fn parse_char_type(&mut self) -> DataType {
+ let length = self.parse_string_type();
+ if length < 0 {
+ DataType::Char(CharType::default())
+ } else {
+ DataType::Char(CharType::new(length as usize).unwrap())
+ }
+ }
+
+ pub fn parse_var_char_type(&mut self) -> DataType {
+ let length = self.parse_string_type();
+ if length < 0 {
+ DataType::VarChar(VarCharType::default())
+ } else {
+ DataType::VarChar(VarCharType::new(length as u32).unwrap())
+ }
+ }
+
+ pub fn parse_binary_type(&mut self) -> DataType {
+ let length = self.parse_string_type();
+ if length < 0 {
+ DataType::Binary(BinaryType::default())
+ } else {
+ DataType::Binary(BinaryType::new(length as usize).unwrap())
+ }
+ }
+
+ pub fn parse_var_binary_type(&mut self) -> DataType {
+ let length = self.parse_string_type();
+ if length < 0 {
+ DataType::VarBinary(VarBinaryType::default())
+ } else {
+ DataType::VarBinary(VarBinaryType::new(length as u32).unwrap())
+ }
+ }
+
+ pub fn parse_decimal_type(&mut self) -> DataType {
+ let mut precision = DecimalType::DEFAULT_PRECISION;
+ let mut scale = DecimalType::DEFAULT_SCALE;
+
+ if self.has_next_token_type(vec![TokenType::BeginParameter]) {
+ self.next_token_type(TokenType::BeginParameter);
+ self.next_token_type(TokenType::LiteralInt);
+ precision = self.token_as_int() as u32;
+
+ if self.has_next_token_type(vec![TokenType::ListSeparator]) {
+ self.next_token_type(TokenType::ListSeparator);
+ self.next_token_type(TokenType::LiteralInt);
+ scale = self.token_as_int() as u32;
+ }
+ self.next_token_type(TokenType::EndParameter);
+ }
+
+ DataType::Decimal(DecimalType::new(precision, scale).unwrap())
+ }
+
+ pub fn parse_double_type(&mut self) -> DataType {
+ if self.has_next_token_keyword(vec![Keyword::PRECISION]) {
+ self.next_token_keyword(Keyword::PRECISION);
+ }
+ DataType::Double(DoubleType::new())
+ }
+
+ pub fn parse_time_type(&mut self) -> DataType {
+ let precision = self.parse_optional_precision(TimeType::DEFAULT_PRECISION);
+
+ if self.has_next_token_keyword(vec![Keyword::WITHOUT]) {
+ self.next_token_keyword(Keyword::WITHOUT);
+ self.next_token_keyword(Keyword::TIME);
+ self.next_token_keyword(Keyword::ZONE);
+ }
+
+ DataType::Time(TimeType::new(precision).unwrap())
+ }
+
+ pub fn parse_timestamp_type(&mut self) -> DataType {
+ let precision = self.parse_optional_precision(TimestampType::DEFAULT_PRECISION);
+
+ if self.has_next_token_keyword(vec![Keyword::WITHOUT]) {
+ self.next_token_keyword(Keyword::WITHOUT);
+ self.next_token_keyword(Keyword::TIME);
+ self.next_token_keyword(Keyword::ZONE);
+ } else if self.has_next_token_keyword(vec![Keyword::WITH]) {
+ self.next_token_keyword(Keyword::WITH);
+ if self.has_next_token_keyword(vec![Keyword::LOCAL]) {
+ self.next_token_keyword(Keyword::LOCAL);
+ self.next_token_keyword(Keyword::TIME);
+ self.next_token_keyword(Keyword::ZONE);
+ return DataType::LocalZonedTimestamp(
+ LocalZonedTimestampType::new(precision).unwrap(),
+ );
+ }
+ }
+ DataType::Timestamp(TimestampType::new(precision).unwrap())
+ }
+
+ pub fn parse_timestamp_ltz_type(&mut self) -> DataType {
+ let precision = self.parse_optional_precision(LocalZonedTimestampType::DEFAULT_PRECISION);
+ DataType::LocalZonedTimestamp(LocalZonedTimestampType::new(precision).unwrap())
+ }
+
+ pub fn parse_optional_precision(&mut self, default_precision: u32) -> u32 {
+ let mut precision = default_precision;
+ if self.has_next_token_type(vec![TokenType::BeginParameter]) {
+ self.next_token_type(TokenType::BeginParameter);
+ self.next_token_type(TokenType::LiteralInt);
+ precision = self.token_as_int() as u32;
+ self.next_token_type(TokenType::EndParameter);
+ }
+ precision
+ }
+}
+
+#[derive(Debug, Default)]
+pub struct DataTypeJsonParser {}
+
+impl DataTypeJsonParser {
+ pub fn is_delimiter(&mut self, character: char) -> bool {
+ character.is_whitespace()
+ || character == CHAR_BEGIN_SUBTYPE
+ || character == CHAR_END_SUBTYPE
+ || character == CHAR_BEGIN_PARAMETER
+ || character == CHAR_END_PARAMETER
+ || character == CHAR_LIST_SEPARATOR
+ || character == CHAR_DOT
+ }
+
+ pub fn is_digit(&mut self, c: char) -> bool {
+ c.is_ascii_digit()
+ }
+
+ pub fn consume_escaped(
+ &mut self,
+ builder: &mut String,
+ chars: &str,
+ mut cursor: usize,
+ delimiter: char,
+ ) -> usize {
+ // skip delimiter
+ cursor += 1;
+
+ while cursor < chars.len() {
+ let cur_char = chars.chars().nth(cursor).unwrap();
+ if cur_char == delimiter
+ && cursor + 1 < chars.len()
+ && chars.chars().nth(cursor + 1).unwrap() == delimiter
+ {
+ // escaping of the escaping char e.g. "'Hello '' World'"
+ cursor += 1;
+ builder.push(cur_char);
+ } else if cur_char == delimiter {
+ break;
+ } else {
+ builder.push(cur_char);
+ }
+ cursor += 1;
+ }
+ cursor
+ }
+
+ pub fn consume_int(&mut self, builder: &mut String, chars: &str, mut cursor: usize) -> usize {
+ while cursor < chars.len() && self.is_digit(chars.chars().nth(cursor).unwrap()) {
+ builder.push(chars.chars().nth(cursor).unwrap());
+ cursor += 1;
+ }
+ cursor - 1
+ }
+
+ pub fn consume_identifier(
+ &mut self,
+ builder: &mut String,
+ chars: &str,
+ mut cursor: usize,
+ ) -> usize {
+ while cursor < chars.len() && !self.is_delimiter(chars.chars().nth(cursor).unwrap()) {
+ builder.push(chars.chars().nth(cursor).unwrap());
+ cursor += 1;
+ }
+ cursor - 1
+ }
+
+ pub fn tokenize(&mut self, chars: &str) -> Vec<Token> {
+ let mut tokens = Vec::new();
+ let mut builder = String::new();
+ let mut cursor = 0;
+ while cursor < chars.len() {
+ let cur_char = chars.chars().nth(cursor).unwrap();
Review Comment:
Repeatedly calling chars.chars().nth(cursor) to get the current character can lead to performance issues in Rust, especially when dealing with large strings. The reason is that each call to chars.chars().nth(cursor) iterates from the beginning of the string until it reaches the cursor position. This means that as the cursor increases, this operation becomes increasingly expensive.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on PR #42:
URL: https://github.com/apache/paimon-rust/pull/42#issuecomment-2280917276
Hi, @SteNicholas, this PR can be closed now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #42:
URL: https://github.com/apache/paimon-rust/pull/42#discussion_r1708772072
##########
crates/paimon/src/spec/types.rs:
##########
@@ -1338,4 +1496,204 @@ mod tests {
));
assert_eq!(row_type.to_string(), "ROW<INTEGER, ARRAY<INTEGER>>");
}
+
+ #[test]
+ fn test_datatype_serialize_deserialize() {
+ // BooleanType
+ let bool_type: BooleanType = BooleanType::with_nullable(true);
+ let serialized: String = serde_json::to_string(&bool_type).unwrap();
+ assert_eq!(serialized, json!("BOOLEAN").to_string());
Review Comment:
Maybe we can compare to `BOOLEAN` directly?
##########
crates/paimon/src/spec/types.rs:
##########
@@ -1338,4 +1496,204 @@ mod tests {
));
assert_eq!(row_type.to_string(), "ROW<INTEGER, ARRAY<INTEGER>>");
}
+
+ #[test]
+ fn test_datatype_serialize_deserialize() {
+ // BooleanType
+ let bool_type: BooleanType = BooleanType::with_nullable(true);
Review Comment:
We don't need to specify the `BooleanType` and `String` here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "JingsongLi (via GitHub)" <gi...@apache.org>.
JingsongLi closed pull request #42: feat: Implement Ser/De for DataType Correct
URL: https://github.com/apache/paimon-rust/pull/42
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat: Implement Ser/De for DataType Correct [paimon-rust]
Posted by "SteNicholas (via GitHub)" <gi...@apache.org>.
SteNicholas commented on code in PR #42:
URL: https://github.com/apache/paimon-rust/pull/42#discussion_r1712619837
##########
crates/paimon/Cargo.toml:
##########
@@ -32,7 +32,7 @@ chrono = { version = "0.4.38", features = ["serde"] }
serde = { version = "1", features = ["derive"] }
serde_bytes = "0.11.15"
serde_json = "1.0.120"
-serde_with = "3.8.3"
+serde_with = "3.9.0"
Review Comment:
Why update this version？
##########
crates/paimon/src/spec/tokens.rs:
##########
@@ -0,0 +1,1246 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::error::DataTypeParsingSnafu;
+use crate::spec::types::DataType;
+use crate::Error;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::{collections::HashSet, fmt, str::FromStr};
+
+use super::{
+ ArrayType, BigIntType, BinaryType, BooleanType, CharType, DataField, DateType, DecimalType,
+ DoubleType, FloatType, IntType, LocalZonedTimestampType, MapType, MultisetType, RowType,
+ SmallIntType, TimeType, TimestampType, TinyIntType, VarBinaryType, VarCharType,
+};
+
+/// --------------------------------------------------------------------------------------------
+/// Tokenizer
+/// --------------------------------------------------------------------------------------------
+
+const CHAR_BEGIN_SUBTYPE: char = '<';
+const CHAR_END_SUBTYPE: char = '>';
+const CHAR_BEGIN_PARAMETER: char = '(';
+const CHAR_END_PARAMETER: char = ')';
+const CHAR_LIST_SEPARATOR: char = ',';
+const CHAR_STRING: char = '\'';
+const CHAR_IDENTIFIER: char = '`';
+const CHAR_DOT: char = '.';
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)]
+pub enum Keyword {
+ CHAR,
+ VARCHAR,
+ STRING,
+ BOOLEAN,
+ BINARY,
+ VARBINARY,
+ BYTES,
+ DECIMAL,
+ NUMERIC,
+ DEC,
+ TINYINT,
+ SMALLINT,
+ INT,
+ INTEGER,
+ BIGINT,
+ FLOAT,
+ DOUBLE,
+ PRECISION,
+ DATE,
+ TIME,
+ WITH,
+ WITHOUT,
+ LOCAL,
+ ZONE,
+ TIMESTAMP,
+ TimestampLtz,
+ INTERVAL,
+ YEAR,
+ MONTH,
+ DAY,
+ HOUR,
+ MINUTE,
+ SECOND,
+ TO,
+ ARRAY,
+ MULTISET,
+ MAP,
+ ROW,
+ NULL,
+ RAW,
+ LEGACY,
+ NOT,
+}
+
+impl fmt::Display for Keyword {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ Keyword::CHAR => write!(f, "CHAR"),
+ Keyword::VARCHAR => write!(f, "VARCHAR"),
+ Keyword::STRING => write!(f, "STRING"),
+ Keyword::BOOLEAN => write!(f, "BOOLEAN"),
+ Keyword::BINARY => write!(f, "BINARY"),
+ Keyword::VARBINARY => write!(f, "VARBINARY"),
+ Keyword::BYTES => write!(f, "BYTES"),
+ Keyword::DECIMAL => write!(f, "DECIMAL"),
+ Keyword::NUMERIC => write!(f, "NUMERIC"),
+ Keyword::DEC => write!(f, "DEC"),
+ Keyword::TINYINT => write!(f, "TINYINT"),
+ Keyword::SMALLINT => write!(f, "SMALLINT"),
+ Keyword::INT => write!(f, "INT"),
+ Keyword::INTEGER => write!(f, "INTEGER"),
+ Keyword::BIGINT => write!(f, "BIGINT"),
+ Keyword::FLOAT => write!(f, "FLOAT"),
+ Keyword::DOUBLE => write!(f, "DOUBLE"),
+ Keyword::PRECISION => write!(f, "PRECISION"),
+ Keyword::DATE => write!(f, "DATE"),
+ Keyword::TIME => write!(f, "TIME"),
+ Keyword::WITH => write!(f, "WITH"),
+ Keyword::WITHOUT => write!(f, "WITHOUT"),
+ Keyword::LOCAL => write!(f, "LOCAL"),
+ Keyword::ZONE => write!(f, "ZONE"),
+ Keyword::TIMESTAMP => write!(f, "TIMESTAMP"),
+ Keyword::TimestampLtz => write!(f, "TIMESTAMP_LTZ"),
+ Keyword::INTERVAL => write!(f, "INTERVAL"),
+ Keyword::YEAR => write!(f, "YEAR"),
+ Keyword::MONTH => write!(f, "MONTH"),
+ Keyword::DAY => write!(f, "DAY"),
+ Keyword::HOUR => write!(f, "HOUR"),
+ Keyword::MINUTE => write!(f, "MINUTE"),
+ Keyword::SECOND => write!(f, "SECOND"),
+ Keyword::TO => write!(f, "TO"),
+ Keyword::ARRAY => write!(f, "ARRAY"),
+ Keyword::MULTISET => write!(f, "MULTISET"),
+ Keyword::MAP => write!(f, "MAP"),
+ Keyword::ROW => write!(f, "ROW"),
+ Keyword::NULL => write!(f, "NULL"),
+ Keyword::RAW => write!(f, "RAW"),
+ Keyword::LEGACY => write!(f, "LEGACY"),
+ Keyword::NOT => write!(f, "NOT"),
+ }
+ }
+}
+
+impl FromStr for Keyword {
+ type Err = ();
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s.to_uppercase().as_str() {
+ "CHAR" => Ok(Keyword::CHAR),
+ "VARCHAR" => Ok(Keyword::VARCHAR),
+ "STRING" => Ok(Keyword::STRING),
+ "BOOLEAN" => Ok(Keyword::BOOLEAN),
+ "BINARY" => Ok(Keyword::BINARY),
+ "VARBINARY" => Ok(Keyword::VARBINARY),
+ "BYTES" => Ok(Keyword::BYTES),
+ "DECIMAL" => Ok(Keyword::DECIMAL),
+ "NUMERIC" => Ok(Keyword::NUMERIC),
+ "DEC" => Ok(Keyword::DEC),
+ "TINYINT" => Ok(Keyword::TINYINT),
+ "SMALLINT" => Ok(Keyword::SMALLINT),
+ "INT" => Ok(Keyword::INT),
+ "INTEGER" => Ok(Keyword::INTEGER),
+ "BIGINT" => Ok(Keyword::BIGINT),
+ "FLOAT" => Ok(Keyword::FLOAT),
+ "DOUBLE" => Ok(Keyword::DOUBLE),
+ "PRECISION" => Ok(Keyword::PRECISION),
+ "DATE" => Ok(Keyword::DATE),
+ "TIME" => Ok(Keyword::TIME),
+ "WITH" => Ok(Keyword::WITH),
+ "WITHOUT" => Ok(Keyword::WITHOUT),
+ "LOCAL" => Ok(Keyword::LOCAL),
+ "ZONE" => Ok(Keyword::ZONE),
+ "TIMESTAMP" => Ok(Keyword::TIMESTAMP),
+ "TIMESTAMP_LTZ" => Ok(Keyword::TimestampLtz),
+ "INTERVAL" => Ok(Keyword::INTERVAL),
+ "YEAR" => Ok(Keyword::YEAR),
+ "MONTH" => Ok(Keyword::MONTH),
+ "DAY" => Ok(Keyword::DAY),
+ "HOUR" => Ok(Keyword::HOUR),
+ "MINUTE" => Ok(Keyword::MINUTE),
+ "SECOND" => Ok(Keyword::SECOND),
+ "TO" => Ok(Keyword::TO),
+ "ARRAY" => Ok(Keyword::ARRAY),
+ "MULTISET" => Ok(Keyword::MULTISET),
+ "MAP" => Ok(Keyword::MAP),
+ "ROW" => Ok(Keyword::ROW),
+ "NULL" => Ok(Keyword::NULL),
+ "RAW" => Ok(Keyword::RAW),
+ "LEGACY" => Ok(Keyword::LEGACY),
+ "NOT" => Ok(Keyword::NOT),
+ _ => Err(()),
+ }
+ }
+}
+
+impl Keyword {
+ pub fn variants() -> HashSet<Keyword> {
+ vec![
+ Keyword::CHAR,
+ Keyword::VARCHAR,
+ Keyword::STRING,
+ Keyword::BOOLEAN,
+ Keyword::BINARY,
+ Keyword::VARBINARY,
+ Keyword::BYTES,
+ Keyword::DECIMAL,
+ Keyword::NUMERIC,
+ Keyword::DEC,
+ Keyword::TINYINT,
+ Keyword::SMALLINT,
+ Keyword::INT,
+ Keyword::INTEGER,
+ Keyword::BIGINT,
+ Keyword::FLOAT,
+ Keyword::DOUBLE,
+ Keyword::PRECISION,
+ Keyword::DATE,
+ Keyword::TIME,
+ Keyword::WITH,
+ Keyword::WITHOUT,
+ Keyword::LOCAL,
+ Keyword::ZONE,
+ Keyword::TIMESTAMP,
+ Keyword::TimestampLtz,
+ Keyword::INTERVAL,
+ Keyword::YEAR,
+ Keyword::MONTH,
+ Keyword::DAY,
+ Keyword::HOUR,
+ Keyword::MINUTE,
+ Keyword::SECOND,
+ Keyword::TO,
+ Keyword::ARRAY,
+ Keyword::MULTISET,
+ Keyword::MAP,
+ Keyword::ROW,
+ Keyword::NULL,
+ Keyword::RAW,
+ Keyword::LEGACY,
+ Keyword::NOT,
+ ]
+ .into_iter()
+ .collect()
+ }
+
+ pub fn is_keyword(s: &str) -> bool {
+ Keyword::variants().iter().any(|v| v.to_string() == s)
+ }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum TokenType {
+ /// e.g. "ROW<"
+ BeginSubtype,
+ /// e.g. "ROW<..>"
+ EndSubtype,
+ /// e.g. "CHAR("
+ BeginParameter,
+ /// e.g. "CHAR(...)"
+ EndParameter,
+ /// e.g. "ROW<INT,"
+ ListSeparator,
+ /// e.g. "ROW<name INT 'Comment'"
+ LiteralString,
+ /// CHAR(12
+ LiteralInt,
+ /// e.g. "CHAR" or "TO"
+ Keyword,
+ /// e.g. "ROW<name" or "myCatalog.myDatabase"
+ Identifier,
+ /// e.g. "myCatalog.myDatabase."
+ IdentifierSeparator,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Token {
+ token_type: TokenType,
+ cursor_position: isize,
+ value: String,
+}
+
+impl Token {
+ pub fn new(token_type: TokenType, cursor_position: isize, value: String) -> Self {
+ Token {
+ token_type,
+ cursor_position,
+ value,
+ }
+ }
+}
+
+pub struct TokenParser {
+ input_string: String,
+ tokens: Vec<Token>,
+ last_valid_token: isize,
+ current_token: isize,
+}
+
+impl TokenParser {
+ pub fn new(input_string: String, tokens: Vec<Token>) -> Self {
+ TokenParser {
+ input_string,
+ tokens,
+ last_valid_token: -1,
+ current_token: -1,
+ }
+ }
+
+ pub fn parse_tokens(&mut self) -> Result<DataType, Error> {
+ let type_data = self.parse_type_with_nullability();
+ if self.has_remaining_tokens() {
+ self.next_token();
+ self.parsing_error(format!("Unexpected token: {}", self.token().value));
+ }
+ Ok(type_data)
+ }
+
+ pub fn last_cursor(&self) -> usize {
+ if self.last_valid_token < 0 {
+ 0
+ } else {
+ self.tokens[self.last_valid_token as usize].cursor_position as usize + 1
+ }
+ }
+
+ pub fn parsing_error(&self, cause: String) -> ! {
+ panic!(
+ "Could not parse type at position {}: {}\nInput type string: {}",
Review Comment:
```suggestion
"Could not parse type at position {}: {}\nInput type: {}",
```
##########
crates/paimon/src/spec/tokens.rs:
##########
@@ -0,0 +1,1246 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::error::DataTypeParsingSnafu;
+use crate::spec::types::DataType;
+use crate::Error;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::{collections::HashSet, fmt, str::FromStr};
+
+use super::{
+ ArrayType, BigIntType, BinaryType, BooleanType, CharType, DataField, DateType, DecimalType,
+ DoubleType, FloatType, IntType, LocalZonedTimestampType, MapType, MultisetType, RowType,
+ SmallIntType, TimeType, TimestampType, TinyIntType, VarBinaryType, VarCharType,
+};
+
+/// --------------------------------------------------------------------------------------------
+/// Tokenizer
+/// --------------------------------------------------------------------------------------------
+
+const CHAR_BEGIN_SUBTYPE: char = '<';
+const CHAR_END_SUBTYPE: char = '>';
+const CHAR_BEGIN_PARAMETER: char = '(';
+const CHAR_END_PARAMETER: char = ')';
+const CHAR_LIST_SEPARATOR: char = ',';
+const CHAR_STRING: char = '\'';
+const CHAR_IDENTIFIER: char = '`';
+const CHAR_DOT: char = '.';
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)]
+pub enum Keyword {
+ CHAR,
+ VARCHAR,
+ STRING,
+ BOOLEAN,
+ BINARY,
+ VARBINARY,
+ BYTES,
+ DECIMAL,
+ NUMERIC,
+ DEC,
+ TINYINT,
+ SMALLINT,
+ INT,
+ INTEGER,
+ BIGINT,
+ FLOAT,
+ DOUBLE,
+ PRECISION,
+ DATE,
+ TIME,
+ WITH,
+ WITHOUT,
+ LOCAL,
+ ZONE,
+ TIMESTAMP,
+ TimestampLtz,
+ INTERVAL,
+ YEAR,
+ MONTH,
+ DAY,
+ HOUR,
+ MINUTE,
+ SECOND,
+ TO,
+ ARRAY,
+ MULTISET,
+ MAP,
+ ROW,
+ NULL,
+ RAW,
+ LEGACY,
+ NOT,
+}
+
+impl fmt::Display for Keyword {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ Keyword::CHAR => write!(f, "CHAR"),
+ Keyword::VARCHAR => write!(f, "VARCHAR"),
+ Keyword::STRING => write!(f, "STRING"),
+ Keyword::BOOLEAN => write!(f, "BOOLEAN"),
+ Keyword::BINARY => write!(f, "BINARY"),
+ Keyword::VARBINARY => write!(f, "VARBINARY"),
+ Keyword::BYTES => write!(f, "BYTES"),
+ Keyword::DECIMAL => write!(f, "DECIMAL"),
+ Keyword::NUMERIC => write!(f, "NUMERIC"),
+ Keyword::DEC => write!(f, "DEC"),
+ Keyword::TINYINT => write!(f, "TINYINT"),
+ Keyword::SMALLINT => write!(f, "SMALLINT"),
+ Keyword::INT => write!(f, "INT"),
+ Keyword::INTEGER => write!(f, "INTEGER"),
+ Keyword::BIGINT => write!(f, "BIGINT"),
+ Keyword::FLOAT => write!(f, "FLOAT"),
+ Keyword::DOUBLE => write!(f, "DOUBLE"),
+ Keyword::PRECISION => write!(f, "PRECISION"),
+ Keyword::DATE => write!(f, "DATE"),
+ Keyword::TIME => write!(f, "TIME"),
+ Keyword::WITH => write!(f, "WITH"),
+ Keyword::WITHOUT => write!(f, "WITHOUT"),
+ Keyword::LOCAL => write!(f, "LOCAL"),
+ Keyword::ZONE => write!(f, "ZONE"),
+ Keyword::TIMESTAMP => write!(f, "TIMESTAMP"),
+ Keyword::TimestampLtz => write!(f, "TIMESTAMP_LTZ"),
+ Keyword::INTERVAL => write!(f, "INTERVAL"),
+ Keyword::YEAR => write!(f, "YEAR"),
+ Keyword::MONTH => write!(f, "MONTH"),
+ Keyword::DAY => write!(f, "DAY"),
+ Keyword::HOUR => write!(f, "HOUR"),
+ Keyword::MINUTE => write!(f, "MINUTE"),
+ Keyword::SECOND => write!(f, "SECOND"),
+ Keyword::TO => write!(f, "TO"),
+ Keyword::ARRAY => write!(f, "ARRAY"),
+ Keyword::MULTISET => write!(f, "MULTISET"),
+ Keyword::MAP => write!(f, "MAP"),
+ Keyword::ROW => write!(f, "ROW"),
+ Keyword::NULL => write!(f, "NULL"),
+ Keyword::RAW => write!(f, "RAW"),
+ Keyword::LEGACY => write!(f, "LEGACY"),
+ Keyword::NOT => write!(f, "NOT"),
+ }
+ }
+}
+
+impl FromStr for Keyword {
+ type Err = ();
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s.to_uppercase().as_str() {
+ "CHAR" => Ok(Keyword::CHAR),
+ "VARCHAR" => Ok(Keyword::VARCHAR),
+ "STRING" => Ok(Keyword::STRING),
+ "BOOLEAN" => Ok(Keyword::BOOLEAN),
+ "BINARY" => Ok(Keyword::BINARY),
+ "VARBINARY" => Ok(Keyword::VARBINARY),
+ "BYTES" => Ok(Keyword::BYTES),
+ "DECIMAL" => Ok(Keyword::DECIMAL),
+ "NUMERIC" => Ok(Keyword::NUMERIC),
+ "DEC" => Ok(Keyword::DEC),
+ "TINYINT" => Ok(Keyword::TINYINT),
+ "SMALLINT" => Ok(Keyword::SMALLINT),
+ "INT" => Ok(Keyword::INT),
+ "INTEGER" => Ok(Keyword::INTEGER),
+ "BIGINT" => Ok(Keyword::BIGINT),
+ "FLOAT" => Ok(Keyword::FLOAT),
+ "DOUBLE" => Ok(Keyword::DOUBLE),
+ "PRECISION" => Ok(Keyword::PRECISION),
+ "DATE" => Ok(Keyword::DATE),
+ "TIME" => Ok(Keyword::TIME),
+ "WITH" => Ok(Keyword::WITH),
+ "WITHOUT" => Ok(Keyword::WITHOUT),
+ "LOCAL" => Ok(Keyword::LOCAL),
+ "ZONE" => Ok(Keyword::ZONE),
+ "TIMESTAMP" => Ok(Keyword::TIMESTAMP),
+ "TIMESTAMP_LTZ" => Ok(Keyword::TimestampLtz),
+ "INTERVAL" => Ok(Keyword::INTERVAL),
+ "YEAR" => Ok(Keyword::YEAR),
+ "MONTH" => Ok(Keyword::MONTH),
+ "DAY" => Ok(Keyword::DAY),
+ "HOUR" => Ok(Keyword::HOUR),
+ "MINUTE" => Ok(Keyword::MINUTE),
+ "SECOND" => Ok(Keyword::SECOND),
+ "TO" => Ok(Keyword::TO),
+ "ARRAY" => Ok(Keyword::ARRAY),
+ "MULTISET" => Ok(Keyword::MULTISET),
+ "MAP" => Ok(Keyword::MAP),
+ "ROW" => Ok(Keyword::ROW),
+ "NULL" => Ok(Keyword::NULL),
+ "RAW" => Ok(Keyword::RAW),
+ "LEGACY" => Ok(Keyword::LEGACY),
+ "NOT" => Ok(Keyword::NOT),
+ _ => Err(()),
+ }
+ }
+}
+
+impl Keyword {
+ pub fn variants() -> HashSet<Keyword> {
+ vec![
+ Keyword::CHAR,
+ Keyword::VARCHAR,
+ Keyword::STRING,
+ Keyword::BOOLEAN,
+ Keyword::BINARY,
+ Keyword::VARBINARY,
+ Keyword::BYTES,
+ Keyword::DECIMAL,
+ Keyword::NUMERIC,
+ Keyword::DEC,
+ Keyword::TINYINT,
+ Keyword::SMALLINT,
+ Keyword::INT,
+ Keyword::INTEGER,
+ Keyword::BIGINT,
+ Keyword::FLOAT,
+ Keyword::DOUBLE,
+ Keyword::PRECISION,
+ Keyword::DATE,
+ Keyword::TIME,
+ Keyword::WITH,
+ Keyword::WITHOUT,
+ Keyword::LOCAL,
+ Keyword::ZONE,
+ Keyword::TIMESTAMP,
+ Keyword::TimestampLtz,
+ Keyword::INTERVAL,
+ Keyword::YEAR,
+ Keyword::MONTH,
+ Keyword::DAY,
+ Keyword::HOUR,
+ Keyword::MINUTE,
+ Keyword::SECOND,
+ Keyword::TO,
+ Keyword::ARRAY,
+ Keyword::MULTISET,
+ Keyword::MAP,
+ Keyword::ROW,
+ Keyword::NULL,
+ Keyword::RAW,
+ Keyword::LEGACY,
+ Keyword::NOT,
+ ]
+ .into_iter()
+ .collect()
+ }
+
+ pub fn is_keyword(s: &str) -> bool {
+ Keyword::variants().iter().any(|v| v.to_string() == s)
+ }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum TokenType {
+ /// e.g. "ROW<"
+ BeginSubtype,
+ /// e.g. "ROW<..>"
+ EndSubtype,
+ /// e.g. "CHAR("
+ BeginParameter,
+ /// e.g. "CHAR(...)"
+ EndParameter,
+ /// e.g. "ROW<INT,"
+ ListSeparator,
+ /// e.g. "ROW<name INT 'Comment'"
+ LiteralString,
+ /// CHAR(12
+ LiteralInt,
+ /// e.g. "CHAR" or "TO"
+ Keyword,
+ /// e.g. "ROW<name" or "myCatalog.myDatabase"
+ Identifier,
+ /// e.g. "myCatalog.myDatabase."
+ IdentifierSeparator,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Token {
+ token_type: TokenType,
+ cursor_position: isize,
+ value: String,
+}
+
+impl Token {
+ pub fn new(token_type: TokenType, cursor_position: isize, value: String) -> Self {
+ Token {
+ token_type,
+ cursor_position,
+ value,
+ }
+ }
+}
+
+pub struct TokenParser {
+ input_string: String,
+ tokens: Vec<Token>,
+ last_valid_token: isize,
+ current_token: isize,
+}
+
+impl TokenParser {
+ pub fn new(input_string: String, tokens: Vec<Token>) -> Self {
+ TokenParser {
+ input_string,
+ tokens,
+ last_valid_token: -1,
+ current_token: -1,
+ }
+ }
+
+ pub fn parse_tokens(&mut self) -> Result<DataType, Error> {
+ let type_data = self.parse_type_with_nullability();
+ if self.has_remaining_tokens() {
+ self.next_token();
+ self.parsing_error(format!("Unexpected token: {}", self.token().value));
+ }
+ Ok(type_data)
+ }
+
+ pub fn last_cursor(&self) -> usize {
+ if self.last_valid_token < 0 {
+ 0
+ } else {
+ self.tokens[self.last_valid_token as usize].cursor_position as usize + 1
+ }
+ }
+
+ pub fn parsing_error(&self, cause: String) -> ! {
+ panic!(
Review Comment:
Any TODO comment？
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@paimon.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org