You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by di...@apache.org on 2023/01/28 04:00:53 UTC
[doris-spark-connector] branch master updated: [feature] support datev2/datatimev2/decimalv3 type (#66)
This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-spark-connector.git
The following commit(s) were added to refs/heads/master by this push:
new b189e5e [feature] support datev2/datatimev2/decimalv3 type (#66)
b189e5e is described below
commit b189e5eb7d9194c628b42ebdcc8d699ec327e63a
Author: gnehil <ad...@gmail.com>
AuthorDate: Sat Jan 28 12:00:48 2023 +0800
[feature] support datev2/datatimev2/decimalv3 type (#66)
* [feature] support datev2/datatimev2/decimalv3 type
* remove debug code
---
.../java/org/apache/doris/spark/serialization/RowBatch.java | 5 +++++
.../main/scala/org/apache/doris/spark/sql/SchemaUtils.scala | 5 +++++
spark-doris-connector/src/main/thrift/doris/Types.thrift | 13 +++++++++++--
3 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/spark-doris-connector/src/main/java/org/apache/doris/spark/serialization/RowBatch.java b/spark-doris-connector/src/main/java/org/apache/doris/spark/serialization/RowBatch.java
index bcc76d5..f04cc5d 100644
--- a/spark-doris-connector/src/main/java/org/apache/doris/spark/serialization/RowBatch.java
+++ b/spark-doris-connector/src/main/java/org/apache/doris/spark/serialization/RowBatch.java
@@ -239,6 +239,9 @@ public class RowBatch {
}
break;
case "DECIMALV2":
+ case "DECIMAL32":
+ case "DECIMAL64":
+ case "DECIMAL128I":
Preconditions.checkArgument(mt.equals(Types.MinorType.DECIMAL),
typeMismatchMessage(currentType, mt));
DecimalVector decimalVector = (DecimalVector) curFieldVector;
@@ -252,7 +255,9 @@ public class RowBatch {
}
break;
case "DATE":
+ case "DATEV2":
case "DATETIME":
+ case "DATETIMEV2":
case "LARGEINT":
case "CHAR":
case "VARCHAR":
diff --git a/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/SchemaUtils.scala b/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/SchemaUtils.scala
index 26d5c82..2caf162 100644
--- a/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/SchemaUtils.scala
+++ b/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/SchemaUtils.scala
@@ -97,13 +97,18 @@ private[spark] object SchemaUtils {
case "FLOAT" => DataTypes.FloatType
case "DOUBLE" => DataTypes.DoubleType
case "DATE" => DataTypes.StringType
+ case "DATEV2" => DataTypes.StringType
case "DATETIME" => DataTypes.StringType
+ case "DATETIMEV2" => DataTypes.StringType
case "BINARY" => DataTypes.BinaryType
case "DECIMAL" => DecimalType(precision, scale)
case "CHAR" => DataTypes.StringType
case "LARGEINT" => DataTypes.StringType
case "VARCHAR" => DataTypes.StringType
case "DECIMALV2" => DecimalType(precision, scale)
+ case "DECIMAL32" => DecimalType(precision, scale)
+ case "DECIMAL64" => DecimalType(precision, scale)
+ case "DECIMAL128I" => DecimalType(precision, scale)
case "TIME" => DataTypes.DoubleType
case "STRING" => DataTypes.StringType
case "HLL" =>
diff --git a/spark-doris-connector/src/main/thrift/doris/Types.thrift b/spark-doris-connector/src/main/thrift/doris/Types.thrift
index 44ce606..a30afde 100644
--- a/spark-doris-connector/src/main/thrift/doris/Types.thrift
+++ b/spark-doris-connector/src/main/thrift/doris/Types.thrift
@@ -66,7 +66,7 @@ enum TPrimitiveType {
DATE,
DATETIME,
BINARY,
- DECIMAL,
+ DECIMAL_DEPRACTED, // not used now, only for place holder
// CHAR(n). Currently only supported in UDAs
CHAR,
LARGEINT,
@@ -79,7 +79,16 @@ enum TPrimitiveType {
MAP,
STRUCT,
STRING,
- ALL
+ ALL,
+ QUANTILE_STATE,
+ DATEV2,
+ DATETIMEV2,
+ TIMEV2,
+ DECIMAL32,
+ DECIMAL64,
+ DECIMAL128I,
+ JSONB,
+ UNSUPPORTED
}
enum TTypeNodeType {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org