You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by dbtsai <gi...@git.apache.org> on 2018/08/13 20:47:14 UTC
[GitHub] spark pull request #22094: [SPARK-25104][SQL]Avro: Validate user specified o...
Github user dbtsai commented on a diff in the pull request:
https://github.com/apache/spark/pull/22094#discussion_r209750701
--- Diff: external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala ---
@@ -72,62 +73,70 @@ class AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable:
private lazy val decimalConversions = new DecimalConversion()
private def newConverter(catalystType: DataType, avroType: Schema): Converter = {
- catalystType match {
- case NullType =>
+ (catalystType, avroType.getType) match {
+ case (NullType, NULL) =>
(getter, ordinal) => null
- case BooleanType =>
+ case (BooleanType, BOOLEAN) =>
(getter, ordinal) => getter.getBoolean(ordinal)
- case ByteType =>
+ case (ByteType, INT) =>
(getter, ordinal) => getter.getByte(ordinal).toInt
- case ShortType =>
+ case (ShortType, INT) =>
(getter, ordinal) => getter.getShort(ordinal).toInt
- case IntegerType =>
+ case (IntegerType, INT) =>
(getter, ordinal) => getter.getInt(ordinal)
- case LongType =>
+ case (LongType, LONG) =>
(getter, ordinal) => getter.getLong(ordinal)
- case FloatType =>
+ case (FloatType, FLOAT) =>
(getter, ordinal) => getter.getFloat(ordinal)
- case DoubleType =>
+ case (DoubleType, DOUBLE) =>
--- End diff --
Do we want to allow users to do casting up from catalystType to avroType? For example, catalystType float to avroType double.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org