You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2023/04/10 01:36:55 UTC
[spark] branch master updated: [SPARK-43073][CONNECT] Add proto data types constants
This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new aa791d93030 [SPARK-43073][CONNECT] Add proto data types constants
aa791d93030 is described below
commit aa791d930303b182ee037aad8b65c6c479ac3bc2
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Mon Apr 10 09:36:28 2023 +0800
[SPARK-43073][CONNECT] Add proto data types constants
### What changes were proposed in this pull request?
Add constants for un-parameterized proto data types
### Why are the changes needed?
avoid recreating them
### Does this PR introduce _any_ user-facing change?
No, dev-only
### How was this patch tested?
existing UT
Closes #40712 from zhengruifeng/connect_proto_types.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
.../connect/common/DataTypeProtoConverter.scala | 84 +++-------------
.../common/LiteralValueProtoConverter.scala | 5 +-
.../spark/sql/connect/common/ProtoDataTypes.scala | 106 +++++++++++++++++++++
3 files changed, 121 insertions(+), 74 deletions(-)
diff --git a/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/common/DataTypeProtoConverter.scala b/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/common/DataTypeProtoConverter.scala
index 28ddbe844d4..c2ef171498d 100644
--- a/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/common/DataTypeProtoConverter.scala
+++ b/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/common/DataTypeProtoConverter.scala
@@ -144,59 +144,23 @@ object DataTypeProtoConverter {
def toConnectProtoType(t: DataType): proto.DataType = {
t match {
- case NullType =>
- proto.DataType
- .newBuilder()
- .setNull(proto.DataType.NULL.getDefaultInstance)
- .build()
+ case NullType => ProtoDataTypes.NullType
- case BooleanType =>
- proto.DataType
- .newBuilder()
- .setBoolean(proto.DataType.Boolean.getDefaultInstance)
- .build()
+ case BooleanType => ProtoDataTypes.BooleanType
- case BinaryType =>
- proto.DataType
- .newBuilder()
- .setBinary(proto.DataType.Binary.getDefaultInstance)
- .build()
+ case BinaryType => ProtoDataTypes.BinaryType
- case ByteType =>
- proto.DataType
- .newBuilder()
- .setByte(proto.DataType.Byte.getDefaultInstance)
- .build()
+ case ByteType => ProtoDataTypes.ByteType
- case ShortType =>
- proto.DataType
- .newBuilder()
- .setShort(proto.DataType.Short.getDefaultInstance)
- .build()
+ case ShortType => ProtoDataTypes.ShortType
- case IntegerType =>
- proto.DataType
- .newBuilder()
- .setInteger(proto.DataType.Integer.getDefaultInstance)
- .build()
+ case IntegerType => ProtoDataTypes.IntegerType
- case LongType =>
- proto.DataType
- .newBuilder()
- .setLong(proto.DataType.Long.getDefaultInstance)
- .build()
+ case LongType => ProtoDataTypes.LongType
- case FloatType =>
- proto.DataType
- .newBuilder()
- .setFloat(proto.DataType.Float.getDefaultInstance)
- .build()
+ case FloatType => ProtoDataTypes.FloatType
- case DoubleType =>
- proto.DataType
- .newBuilder()
- .setDouble(proto.DataType.Double.getDefaultInstance)
- .build()
+ case DoubleType => ProtoDataTypes.DoubleType
case DecimalType.Fixed(precision, scale) =>
proto.DataType
@@ -205,11 +169,7 @@ object DataTypeProtoConverter {
proto.DataType.Decimal.newBuilder().setPrecision(precision).setScale(scale).build())
.build()
- case StringType =>
- proto.DataType
- .newBuilder()
- .setString(proto.DataType.String.getDefaultInstance)
- .build()
+ case StringType => ProtoDataTypes.StringType
case CharType(length) =>
proto.DataType
@@ -223,29 +183,13 @@ object DataTypeProtoConverter {
.setVarChar(proto.DataType.VarChar.newBuilder().setLength(length).build())
.build()
- case DateType =>
- proto.DataType
- .newBuilder()
- .setDate(proto.DataType.Date.getDefaultInstance)
- .build()
+ case DateType => ProtoDataTypes.DateType
- case TimestampType =>
- proto.DataType
- .newBuilder()
- .setTimestamp(proto.DataType.Timestamp.getDefaultInstance)
- .build()
+ case TimestampType => ProtoDataTypes.TimestampType
- case TimestampNTZType =>
- proto.DataType
- .newBuilder()
- .setTimestampNtz(proto.DataType.TimestampNTZ.getDefaultInstance)
- .build()
+ case TimestampNTZType => ProtoDataTypes.TimestampNTZType
- case CalendarIntervalType =>
- proto.DataType
- .newBuilder()
- .setCalendarInterval(proto.DataType.CalendarInterval.getDefaultInstance)
- .build()
+ case CalendarIntervalType => ProtoDataTypes.CalendarIntervalType
case YearMonthIntervalType(startField, endField) =>
proto.DataType
diff --git a/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/common/LiteralValueProtoConverter.scala b/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/common/LiteralValueProtoConverter.scala
index 90040c294ef..3e7892bbbf5 100644
--- a/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/common/LiteralValueProtoConverter.scala
+++ b/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/common/LiteralValueProtoConverter.scala
@@ -35,9 +35,6 @@ import org.apache.spark.unsafe.types.CalendarInterval
object LiteralValueProtoConverter {
- private lazy val nullType =
- proto.DataType.newBuilder().setNull(proto.DataType.NULL.getDefaultInstance).build()
-
/**
* Transforms literal value to the `proto.Expression.Literal.Builder`.
*
@@ -95,7 +92,7 @@ object LiteralValueProtoConverter {
case v: Array[_] => builder.setArray(arrayBuilder(v))
case v: CalendarInterval =>
builder.setCalendarInterval(calendarIntervalBuilder(v.months, v.days, v.microseconds))
- case null => builder.setNull(nullType)
+ case null => builder.setNull(ProtoDataTypes.NullType)
case _ => throw new UnsupportedOperationException(s"literal $literal not supported (yet).")
}
}
diff --git a/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/common/ProtoDataTypes.scala b/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/common/ProtoDataTypes.scala
new file mode 100644
index 00000000000..19890558ab2
--- /dev/null
+++ b/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/common/ProtoDataTypes.scala
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connect.common
+
+import org.apache.spark.connect.proto
+
+private[connect] object ProtoDataTypes {
+
+ val NullType: proto.DataType = proto.DataType
+ .newBuilder()
+ .setNull(proto.DataType.NULL.getDefaultInstance)
+ .build()
+
+ val BooleanType: proto.DataType =
+ proto.DataType
+ .newBuilder()
+ .setBoolean(proto.DataType.Boolean.getDefaultInstance)
+ .build()
+
+ val BinaryType: proto.DataType =
+ proto.DataType
+ .newBuilder()
+ .setBinary(proto.DataType.Binary.getDefaultInstance)
+ .build()
+
+ val ByteType: proto.DataType =
+ proto.DataType
+ .newBuilder()
+ .setByte(proto.DataType.Byte.getDefaultInstance)
+ .build()
+
+ val ShortType: proto.DataType =
+ proto.DataType
+ .newBuilder()
+ .setShort(proto.DataType.Short.getDefaultInstance)
+ .build()
+
+ val IntegerType: proto.DataType =
+ proto.DataType
+ .newBuilder()
+ .setInteger(proto.DataType.Integer.getDefaultInstance)
+ .build()
+
+ val LongType: proto.DataType =
+ proto.DataType
+ .newBuilder()
+ .setLong(proto.DataType.Long.getDefaultInstance)
+ .build()
+
+ val FloatType: proto.DataType =
+ proto.DataType
+ .newBuilder()
+ .setFloat(proto.DataType.Float.getDefaultInstance)
+ .build()
+
+ val DoubleType: proto.DataType =
+ proto.DataType
+ .newBuilder()
+ .setDouble(proto.DataType.Double.getDefaultInstance)
+ .build()
+
+ val StringType: proto.DataType =
+ proto.DataType
+ .newBuilder()
+ .setString(proto.DataType.String.getDefaultInstance)
+ .build()
+
+ val DateType: proto.DataType =
+ proto.DataType
+ .newBuilder()
+ .setDate(proto.DataType.Date.getDefaultInstance)
+ .build()
+
+ val TimestampType: proto.DataType =
+ proto.DataType
+ .newBuilder()
+ .setTimestamp(proto.DataType.Timestamp.getDefaultInstance)
+ .build()
+
+ val TimestampNTZType: proto.DataType =
+ proto.DataType
+ .newBuilder()
+ .setTimestampNtz(proto.DataType.TimestampNTZ.getDefaultInstance)
+ .build()
+
+ val CalendarIntervalType: proto.DataType =
+ proto.DataType
+ .newBuilder()
+ .setCalendarInterval(proto.DataType.CalendarInterval.getDefaultInstance)
+ .build()
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org